diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 4a4a470c0d1f..75b95d956e5e 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -97,7 +97,6 @@ public class HiveConf extends Configuration { private static byte[] confVarByteArray = null; private static final Map vars = new HashMap(); - private static final Map metaConfs = new HashMap(); private final List restrictList = new ArrayList(); private final Set hiddenSet = new HashSet(); private final Set lockedSet = new HashSet<>(); @@ -252,118 +251,11 @@ private static URL checkConfigFile(File f) { } } - - - @InterfaceAudience.Private public static final String PREFIX_LLAP = "llap."; @InterfaceAudience.Private public static final String PREFIX_HIVE_LLAP = "hive.llap."; - /** - * Metastore related options that the db is initialized against. When a conf - * var in this is list is changed, the metastore instance for the CLI will - * be recreated so that the change will take effect. - */ - public static final HiveConf.ConfVars[] metaVars = { - HiveConf.ConfVars.METASTORE_WAREHOUSE, - HiveConf.ConfVars.REPL_DIR, - HiveConf.ConfVars.METASTORE_URIS, - HiveConf.ConfVars.METASTORE_SELECTION, - HiveConf.ConfVars.METASTORE_SERVER_PORT, - HiveConf.ConfVars.METASTORE_THRIFT_CONNECTION_RETRIES, - HiveConf.ConfVars.METASTORE_THRIFT_FAILURE_RETRIES, - HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, - HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT, - HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME, - HiveConf.ConfVars.METASTORE_PWD, - HiveConf.ConfVars.METASTORE_CONNECT_URL_HOOK, - HiveConf.ConfVars.METASTORE_CONNECT_URL_KEY, - HiveConf.ConfVars.METASTORE_SERVER_MIN_THREADS, - HiveConf.ConfVars.METASTORE_SERVER_MAX_THREADS, - HiveConf.ConfVars.METASTORE_INT_ORIGINAL, - HiveConf.ConfVars.METASTORE_INT_ARCHIVED, - HiveConf.ConfVars.METASTORE_INT_EXTRACTED, - HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE, - HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL, - HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, - HiveConf.ConfVars.METASTORE_TOKEN_SIGNATURE, - HiveConf.ConfVars.METASTORE_CACHE_PINOBJTYPES, - HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, - HiveConf.ConfVars.METASTORE_VALIDATE_TABLES, - HiveConf.ConfVars.METASTORE_DATANUCLEUS_INIT_COL_INFO, - HiveConf.ConfVars.METASTORE_VALIDATE_COLUMNS, - HiveConf.ConfVars.METASTORE_VALIDATE_CONSTRAINTS, - HiveConf.ConfVars.METASTORE_STORE_MANAGER_TYPE, - HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL, - HiveConf.ConfVars.METASTORE_TRANSACTION_ISOLATION, - HiveConf.ConfVars.METASTORE_CACHE_LEVEL2, - HiveConf.ConfVars.METASTORE_CACHE_LEVEL2_TYPE, - HiveConf.ConfVars.METASTORE_IDENTIFIER_FACTORY, - HiveConf.ConfVars.METASTORE_PLUGIN_REGISTRY_BUNDLE_CHECK, - HiveConf.ConfVars.METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS, - HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX, - HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, - HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS, - HiveConf.ConfVars.METASTORE_EVENT_CLEAN_FREQ, - HiveConf.ConfVars.METASTORE_EVENT_EXPIRY_DURATION, - HiveConf.ConfVars.METASTORE_EVENT_MESSAGE_FACTORY, - HiveConf.ConfVars.METASTORE_FILTER_HOOK, - HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, - HiveConf.ConfVars.METASTORE_END_FUNCTION_LISTENERS, - HiveConf.ConfVars.METASTORE_PART_INHERIT_TBL_PROPS, - HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_OBJECTS_MAX, - HiveConf.ConfVars.METASTORE_INIT_HOOKS, - HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS, - HiveConf.ConfVars.HMS_HANDLER_ATTEMPTS, - HiveConf.ConfVars.HMS_HANDLER_INTERVAL, - HiveConf.ConfVars.HMS_HANDLER_FORCE_RELOAD_CONF, - HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN, - HiveConf.ConfVars.METASTORE_ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS, - HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES, - HiveConf.ConfVars.USERS_IN_ADMIN_ROLE, - HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, - HiveConf.ConfVars.HIVE_TXN_MANAGER, - HiveConf.ConfVars.HIVE_TXN_TIMEOUT, - HiveConf.ConfVars.HIVE_TXN_OPERATIONAL_PROPERTIES, - HiveConf.ConfVars.HIVE_TXN_HEARTBEAT_THREADPOOL_SIZE, - HiveConf.ConfVars.HIVE_TXN_MAX_OPEN_BATCH, - HiveConf.ConfVars.HIVE_TXN_RETRYABLE_SQLEX_REGEX, - HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_TUNER, - HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_DENSITY_FUNCTION, - HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_ENABLED, - HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_SIZE, - HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_PARTITIONS, - HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_FPP, - HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_VARIANCE, - HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_TTL, - HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_WRITER_WAIT, - HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_READER_WAIT, - HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_FULL, - HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_CLEAN_UNTIL, - HiveConf.ConfVars.METASTORE_FASTPATH, - HiveConf.ConfVars.METASTORE_HBASE_FILE_METADATA_THREADS, - HiveConf.ConfVars.METASTORE_WM_DEFAULT_POOL_SIZE - }; - - /** - * User configurable Metastore vars - */ - static final HiveConf.ConfVars[] metaConfVars = { - HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL, - HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL_DDL, - HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT, - HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN, - HiveConf.ConfVars.METASTORE_CAPABILITY_CHECK, - HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES - }; - - static { - for (ConfVars confVar : metaConfVars) { - metaConfs.put(confVar.varname, confVar); - } - } - public static final String HIVE_LLAP_DAEMON_SERVICE_PRINCIPAL_NAME = "hive.llap.daemon.service.principal"; public static final String HIVE_SERVER2_AUTHENTICATION_LDAP_USERMEMBERSHIPKEY_NAME = "hive.server2.authentication.ldap.userMembershipKey"; @@ -6466,10 +6358,6 @@ public static ConfVars getConfVars(String name) { return vars.get(name); } - public static ConfVars getMetaConf(String name) { - return metaConfs.get(name); - } - public String getVar(ConfVars var) { return getVar(this, var); } @@ -6665,12 +6553,6 @@ private void initialize(Class cls) { // Overlay the values of any system properties and manual overrides applySystemProperties(); - if ((this.get("hive.metastore.ds.retry.attempts") != null) || - this.get("hive.metastore.ds.retry.interval") != null) { - LOG.warn("DEPRECATED: hive.metastore.ds.retry.* no longer has any effect. " + - "Use hive.hmshandler.retry.* instead"); - } - // if the running class was loaded directly (through eclipse) rather than through a // jar then this would be needed if (hiveJar == null) { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java index 1fc3663e75bd..0d61eaf2573c 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.security.HadoopDefaultAuthenticator; import org.apache.hadoop.hive.ql.security.authorization.plugin.DisallowTransformHook; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException; @@ -126,8 +127,8 @@ private void verifyParamSettability(List settableParams, HiveConf proces assertConfModificationException(processedConf, "dummy.param"); // does not make sense to have any of the metastore config variables to be // modifiable - for (ConfVars metaVar : HiveConf.metaVars) { - assertConfModificationException(processedConf, metaVar.varname); + for (MetastoreConf.ConfVars metaVar : MetastoreConf.metaVars) { + assertConfModificationException(processedConf, metaVar.getHiveName()); } } diff --git a/ql/src/test/queries/clientpositive/alter_table_column_stats.q b/ql/src/test/queries/clientpositive/alter_table_column_stats.q index cd699b32dc33..3440758c500d 100644 --- a/ql/src/test/queries/clientpositive/alter_table_column_stats.q +++ b/ql/src/test/queries/clientpositive/alter_table_column_stats.q @@ -266,8 +266,8 @@ use default; drop database statsdb1; drop database statsdb2; --- Test for external tables with metastore.try.direct.sql.ddl as false -set metaconf:metastore.try.direct.sql.ddl=false; +-- Test for external tables with metastore.try.direct.sql as false +set metaconf:metastore.try.direct.sql=false; drop database if exists statsdb1; create database statsdb1; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 85b37d67889d..16ccf508a43d 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -250,7 +250,6 @@ public String toString() { ConfVars.METASTORE_PARTITIONS_PARAMETERS_EXCLUDE_PATTERN, // Add metaConfVars here as well ConfVars.TRY_DIRECT_SQL, - ConfVars.TRY_DIRECT_SQL_DDL, ConfVars.CLIENT_SOCKET_TIMEOUT, ConfVars.PARTITION_NAME_WHITELIST_PATTERN, ConfVars.PARTITION_ORDER_EXPR, @@ -264,7 +263,6 @@ public String toString() { */ public static final MetastoreConf.ConfVars[] metaConfVars = { ConfVars.TRY_DIRECT_SQL, - ConfVars.TRY_DIRECT_SQL_DDL, ConfVars.CLIENT_SOCKET_TIMEOUT, ConfVars.PARTITION_NAME_WHITELIST_PATTERN, ConfVars.PARTITION_ORDER_EXPR, @@ -1663,12 +1661,6 @@ public enum ConfVars { "work for all queries on your datastore. If all SQL queries fail (for example, your\n" + "metastore is backed by MongoDB), you might want to disable this to save the\n" + "try-and-fall-back cost."), - TRY_DIRECT_SQL_DDL("metastore.try.direct.sql.ddl", "hive.metastore.try.direct.sql.ddl", true, - "Same as hive.metastore.try.direct.sql, for read statements within a transaction that\n" + - "modifies metastore data. Due to non-standard behavior in Postgres, if a direct SQL\n" + - "select query has incorrect syntax or something similar inside a transaction, the\n" + - "entire transaction will fail and fall-back to DataNucleus will not be possible. You\n" + - "should disable the usage of direct SQL inside transactions if that happens in your case."), TXN_MAX_OPEN_BATCH("metastore.txn.max.open.batch", "hive.txn.max.open.batch", 1000, "Maximum number of transactions that can be fetched in one call to open_txns().\n" + "This controls how many transactions streaming agents such as Flume or Storm open\n" + diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index c9856fcd9125..e006279f2496 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -4174,11 +4174,7 @@ public GetHelper(String catalogName, String dbName, String tblName, this.doTrace = LOG.isDebugEnabled(); this.isInTxn = isActiveTransaction(); - // SQL usage inside a larger transaction (e.g. droptable) may not be desirable because - // some databases (e.g. Postgres) abort the entire transaction when any query fails, so - // the fallback from failed SQL to JDO is not possible. - boolean isConfigEnabled = MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL) - && (MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL_DDL) || !isInTxn); + boolean isConfigEnabled = MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL); if (isConfigEnabled && directSql == null) { directSql = new MetaStoreDirectSql(pm, getConf(), ""); } diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjectionAndFilterSpecs.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjectionAndFilterSpecs.java index 7a37f6032b4a..f284849b73f2 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjectionAndFilterSpecs.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjectionAndFilterSpecs.java @@ -113,7 +113,6 @@ public void setup() throws TException { tbl = client.getTable(dbName, tblName); // set directSQL to true explicitly client.setMetaConf(ConfVars.TRY_DIRECT_SQL.getVarname(), "true"); - client.setMetaConf(ConfVars.TRY_DIRECT_SQL_DDL.getVarname(), "true"); } @After diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java index 5963cb020ab9..c8b3d207cdf5 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java @@ -60,7 +60,6 @@ public static void startMetaStoreServer() throws Exception { Configuration metastoreConf = MetastoreConf.newMetastoreConf(); MetastoreConf.setClass(metastoreConf, ConfVars.EXPRESSION_PROXY_CLASS, MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class); - MetastoreConf.setBoolVar(metastoreConf, ConfVars.TRY_DIRECT_SQL_DDL, false); MetaStoreTestUtils.setConfForStandloneMode(metastoreConf); int msPort = MetaStoreServerUtils.startMetaStore(metastoreConf); conf = MetastoreConf.newMetastoreConf(); @@ -97,14 +96,6 @@ public void testGetMetaConfDefaultEmptyString() throws TException { assertEquals(expected, actual); } - @Test - public void testGetMetaConfOverridden() throws TException { - ConfVars metaConfVar = ConfVars.TRY_DIRECT_SQL_DDL; - String expected = "false"; - String actual = hmsc.getMetaConf(metaConfVar.toString()); - assertEquals(expected, actual); - } - @Test public void testGetMetaConfUnknownPreperty() throws TException { String unknownPropertyName = "hive.meta.foo.bar"; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java index 3f7ec97b79e7..bf09765634e4 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java @@ -1198,7 +1198,6 @@ public void testAlterTableAlreadyExists() throws Exception { @Test public void testAlterTableExpectedPropertyMatch() throws Exception { Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(), ConfVars.TRY_DIRECT_SQL)); - Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(), ConfVars.TRY_DIRECT_SQL_DDL)); Table originalTable = testTables[0]; EnvironmentContext context = new EnvironmentContext(); @@ -1213,7 +1212,6 @@ public void testAlterTableExpectedPropertyMatch() throws Exception { @Test(expected = MetaException.class) public void testAlterTableExpectedPropertyDifferent() throws Exception { Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(), ConfVars.TRY_DIRECT_SQL)); - Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(), ConfVars.TRY_DIRECT_SQL_DDL)); Table originalTable = testTables[0]; EnvironmentContext context = new EnvironmentContext(); @@ -1234,7 +1232,6 @@ public void testAlterTableExpectedPropertyDifferent() throws Exception { @Test public void testAlterTableExpectedPropertyConcurrent() throws Exception { Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(), ConfVars.TRY_DIRECT_SQL)); - Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(), ConfVars.TRY_DIRECT_SQL_DDL)); Table originalTable = testTables[0]; originalTable.getParameters().put("snapshot", "0");