diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminImportTableIntegrationTestWithJdbcDatabase.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminImportTableIntegrationTestWithJdbcDatabase.java index d589c140cc..3757b28ba0 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminImportTableIntegrationTestWithJdbcDatabase.java +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminImportTableIntegrationTestWithJdbcDatabase.java @@ -45,12 +45,12 @@ public void afterAll() { @Override protected List createExistingDatabaseWithAllDataTypes() throws SQLException { - return testUtils.createExistingDatabaseWithAllDataTypes(getNamespace()); + return testUtils.createExistingDatabaseWithAllDataTypes(namespace); } @Override protected void dropNonImportableTable(String table) throws SQLException { - testUtils.dropTable(getNamespace(), table); + testUtils.dropTable(namespace, table); } @SuppressWarnings("unused") diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminImportTableWithMetadataDecouplingIntegrationTestWithJdbcDatabase.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminImportTableWithMetadataDecouplingIntegrationTestWithJdbcDatabase.java new file mode 100644 index 0000000000..486d46c711 --- /dev/null +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminImportTableWithMetadataDecouplingIntegrationTestWithJdbcDatabase.java @@ -0,0 +1,89 @@ +package com.scalar.db.storage.jdbc; + +import com.scalar.db.api.DistributedStorageAdminImportTableIntegrationTestBase.TestData; +import com.scalar.db.config.DatabaseConfig; +import com.scalar.db.exception.storage.ExecutionException; +import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminImportTableWithMetadataDecouplingIntegrationTestBase; +import java.sql.SQLException; +import java.util.List; +import java.util.Properties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; +import org.junit.jupiter.api.condition.EnabledIf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@DisabledIf("com.scalar.db.storage.jdbc.JdbcEnv#isOracle") +public class ConsensusCommitAdminImportTableWithMetadataDecouplingIntegrationTestWithJdbcDatabase + extends ConsensusCommitAdminImportTableWithMetadataDecouplingIntegrationTestBase { + private static final Logger logger = + LoggerFactory.getLogger( + ConsensusCommitAdminImportTableWithMetadataDecouplingIntegrationTestWithJdbcDatabase + .class); + + private JdbcAdminImportTestUtils testUtils; + + @Override + protected Properties getProps(String testName) { + Properties properties = JdbcEnv.getProperties(testName); + + // Set the isolation level for consistency reads for virtual tables + RdbEngineStrategy rdbEngine = + RdbEngineFactory.create(new JdbcConfig(new DatabaseConfig(properties))); + properties.setProperty( + JdbcConfig.ISOLATION_LEVEL, + JdbcTestUtils.getIsolationLevel( + rdbEngine.getMinimumIsolationLevelForConsistentVirtualTableRead()) + .name()); + + testUtils = new JdbcAdminImportTestUtils(properties); + return properties; + } + + @Override + public void afterAll() { + try { + super.afterAll(); + } catch (Exception e) { + logger.warn("Failed to call super.afterAll", e); + } + + try { + if (testUtils != null) { + testUtils.close(); + } + } catch (Exception e) { + logger.warn("Failed to close test utils", e); + } + } + + @Override + protected List createExistingDatabaseWithAllDataTypes() throws SQLException { + return testUtils.createExistingDatabaseWithAllDataTypes(namespace); + } + + @Override + protected void dropNonImportableTable(String table) throws SQLException { + testUtils.dropTable(namespace, table); + } + + @SuppressWarnings("unused") + private boolean isSqlite() { + return JdbcEnv.isSqlite(); + } + + @Test + @Override + @DisabledIf("isSqlite") + public void importTable_ShouldWorkProperly() throws Exception { + super.importTable_ShouldWorkProperly(); + } + + @Test + @Override + @EnabledIf("isSqlite") + public void importTable_ForUnsupportedDatabase_ShouldThrowUnsupportedOperationException() + throws ExecutionException { + super.importTable_ForUnsupportedDatabase_ShouldThrowUnsupportedOperationException(); + } +} diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitImportTableIntegrationTestWithJdbcDatabase.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitImportTableIntegrationTestWithJdbcDatabase.java new file mode 100644 index 0000000000..c99cc2dcbb --- /dev/null +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitImportTableIntegrationTestWithJdbcDatabase.java @@ -0,0 +1,38 @@ +package com.scalar.db.storage.jdbc; + +import com.scalar.db.config.DatabaseConfig; +import com.scalar.db.transaction.consensuscommit.ConsensusCommitImportTableIntegrationTestBase; +import com.scalar.db.util.AdminTestUtils; +import java.util.Properties; +import org.junit.jupiter.api.condition.DisabledIf; + +@DisabledIf("isSqliteOrOracle") +public class ConsensusCommitImportTableIntegrationTestWithJdbcDatabase + extends ConsensusCommitImportTableIntegrationTestBase { + + @Override + protected Properties getProperties(String testName) { + Properties properties = ConsensusCommitJdbcEnv.getProperties(testName); + + // Set the isolation level for consistency reads for virtual tables + RdbEngineStrategy rdbEngine = + RdbEngineFactory.create(new JdbcConfig(new DatabaseConfig(properties))); + properties.setProperty( + JdbcConfig.ISOLATION_LEVEL, + JdbcTestUtils.getIsolationLevel( + rdbEngine.getMinimumIsolationLevelForConsistentVirtualTableRead()) + .name()); + + return properties; + } + + @Override + protected AdminTestUtils getAdminTestUtils(String testName) { + return new JdbcAdminTestUtils(getProperties(testName)); + } + + @SuppressWarnings("unused") + private static boolean isSqliteOrOracle() { + return JdbcEnv.isSqlite() || JdbcEnv.isOracle(); + } +} diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitImportTableWithMetadataDecouplingIntegrationTestWithJdbcDatabase.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitImportTableWithMetadataDecouplingIntegrationTestWithJdbcDatabase.java new file mode 100644 index 0000000000..208d8c2cf7 --- /dev/null +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitImportTableWithMetadataDecouplingIntegrationTestWithJdbcDatabase.java @@ -0,0 +1,38 @@ +package com.scalar.db.storage.jdbc; + +import com.scalar.db.config.DatabaseConfig; +import com.scalar.db.transaction.consensuscommit.ConsensusCommitImportTableWithMetadataDecouplingIntegrationTestBase; +import com.scalar.db.util.AdminTestUtils; +import java.util.Properties; +import org.junit.jupiter.api.condition.DisabledIf; + +@DisabledIf("isSqliteOrOracle") +public class ConsensusCommitImportTableWithMetadataDecouplingIntegrationTestWithJdbcDatabase + extends ConsensusCommitImportTableWithMetadataDecouplingIntegrationTestBase { + + @Override + protected Properties getProperties(String testName) { + Properties properties = ConsensusCommitJdbcEnv.getProperties(testName); + + // Set the isolation level for consistency reads for virtual tables + RdbEngineStrategy rdbEngine = + RdbEngineFactory.create(new JdbcConfig(new DatabaseConfig(properties))); + properties.setProperty( + JdbcConfig.ISOLATION_LEVEL, + JdbcTestUtils.getIsolationLevel( + rdbEngine.getMinimumIsolationLevelForConsistentVirtualTableRead()) + .name()); + + return properties; + } + + @Override + protected AdminTestUtils getAdminTestUtils(String testName) { + return new JdbcAdminTestUtils(getProperties(testName)); + } + + @SuppressWarnings("unused") + private static boolean isSqliteOrOracle() { + return JdbcEnv.isSqlite() || JdbcEnv.isOracle(); + } +} diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitSpecificWithMetadataDecouplingIntegrationTestWithJdbcDatabase.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitSpecificWithMetadataDecouplingIntegrationTestWithJdbcDatabase.java new file mode 100644 index 0000000000..50c97c4e57 --- /dev/null +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitSpecificWithMetadataDecouplingIntegrationTestWithJdbcDatabase.java @@ -0,0 +1,27 @@ +package com.scalar.db.storage.jdbc; + +import com.scalar.db.config.DatabaseConfig; +import com.scalar.db.transaction.consensuscommit.ConsensusCommitSpecificWithMetadataDecouplingIntegrationTestBase; +import java.util.Properties; +import org.junit.jupiter.api.condition.DisabledIf; + +@DisabledIf("com.scalar.db.storage.jdbc.JdbcEnv#isOracle") +public class ConsensusCommitSpecificWithMetadataDecouplingIntegrationTestWithJdbcDatabase + extends ConsensusCommitSpecificWithMetadataDecouplingIntegrationTestBase { + + @Override + protected Properties getProperties(String testName) { + Properties properties = ConsensusCommitJdbcEnv.getProperties(testName); + + // Set the isolation level for consistency reads for virtual tables + RdbEngineStrategy rdbEngine = + RdbEngineFactory.create(new JdbcConfig(new DatabaseConfig(properties))); + properties.setProperty( + JdbcConfig.ISOLATION_LEVEL, + JdbcTestUtils.getIsolationLevel( + rdbEngine.getMinimumIsolationLevelForConsistentVirtualTableRead()) + .name()); + + return properties; + } +} diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitWithMetadataDecouplingIntegrationTestWithJdbcDatabase.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitWithMetadataDecouplingIntegrationTestWithJdbcDatabase.java new file mode 100644 index 0000000000..931a6d8fd4 --- /dev/null +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitWithMetadataDecouplingIntegrationTestWithJdbcDatabase.java @@ -0,0 +1,27 @@ +package com.scalar.db.storage.jdbc; + +import com.scalar.db.config.DatabaseConfig; +import com.scalar.db.transaction.consensuscommit.ConsensusCommitWithMetadataDecouplingIntegrationTestBase; +import java.util.Properties; +import org.junit.jupiter.api.condition.DisabledIf; + +@DisabledIf("com.scalar.db.storage.jdbc.JdbcEnv#isOracle") +public class ConsensusCommitWithMetadataDecouplingIntegrationTestWithJdbcDatabase + extends ConsensusCommitWithMetadataDecouplingIntegrationTestBase { + + @Override + protected Properties getProps(String testName) { + Properties properties = ConsensusCommitJdbcEnv.getProperties(testName); + + // Set the isolation level for consistency reads for virtual tables + RdbEngineStrategy rdbEngine = + RdbEngineFactory.create(new JdbcConfig(new DatabaseConfig(properties))); + properties.setProperty( + JdbcConfig.ISOLATION_LEVEL, + JdbcTestUtils.getIsolationLevel( + rdbEngine.getMinimumIsolationLevelForConsistentVirtualTableRead()) + .name()); + + return properties; + } +} diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcSchemaLoaderImportIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcSchemaLoaderImportIntegrationTest.java index 625a5c2697..be99398212 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcSchemaLoaderImportIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcSchemaLoaderImportIntegrationTest.java @@ -11,7 +11,6 @@ import java.util.Properties; import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledIf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -179,19 +178,6 @@ protected void dropNonImportableTable(String namespace, String table) throws Exc testUtils.dropTable(namespace, table); } - @Test - @Override - public void importTables_ImportableTablesGiven_ShouldImportProperly() throws Exception { - super.importTables_ImportableTablesGiven_ShouldImportProperly(); - } - - @Test - @Override - public void importTables_ImportableTablesAndNonRelatedSameNameTableGiven_ShouldImportProperly() - throws Exception { - super.importTables_ImportableTablesAndNonRelatedSameNameTableGiven_ShouldImportProperly(); - } - @AfterAll @Override public void afterAll() { diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcSchemaLoaderImportWithMetadataDecouplingIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcSchemaLoaderImportWithMetadataDecouplingIntegrationTest.java new file mode 100644 index 0000000000..3369fd3344 --- /dev/null +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcSchemaLoaderImportWithMetadataDecouplingIntegrationTest.java @@ -0,0 +1,222 @@ +package com.scalar.db.storage.jdbc; + +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.Uninterruptibles; +import com.scalar.db.api.TableMetadata; +import com.scalar.db.config.DatabaseConfig; +import com.scalar.db.io.DataType; +import com.scalar.db.schemaloader.SchemaLoaderImportWithMetadataDecouplingIntegrationTestBase; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.condition.DisabledIf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@DisabledIf("isSqliteOrOracle") +public class JdbcSchemaLoaderImportWithMetadataDecouplingIntegrationTest + extends SchemaLoaderImportWithMetadataDecouplingIntegrationTestBase { + + private static final Logger logger = + LoggerFactory.getLogger(JdbcSchemaLoaderImportWithMetadataDecouplingIntegrationTest.class); + + private JdbcAdminImportTestUtils testUtils; + private RdbEngineStrategy rdbEngine; + + @Override + protected Properties getProperties(String testName) { + Properties properties = JdbcEnv.getProperties(testName); + JdbcConfig config = new JdbcConfig(new DatabaseConfig(properties)); + rdbEngine = RdbEngineFactory.create(config); + testUtils = new JdbcAdminImportTestUtils(properties); + + // Set the isolation level for consistency reads for virtual tables + properties.setProperty( + JdbcConfig.ISOLATION_LEVEL, + JdbcTestUtils.getIsolationLevel( + rdbEngine.getMinimumIsolationLevelForConsistentVirtualTableRead()) + .name()); + + return properties; + } + + @SuppressFBWarnings("SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE") + @Override + protected void createImportableTable(String namespace, String table) throws Exception { + String sql; + + if (JdbcTestUtils.isMysql(rdbEngine)) { + sql = + "CREATE TABLE " + + rdbEngine.encloseFullTableName(namespace, table) + + "(" + + rdbEngine.enclose("pk") + + " CHAR(8)," + + rdbEngine.enclose("col1") + + " CHAR(8)," + + rdbEngine.enclose("col2") + + " DATETIME," + + "PRIMARY KEY(" + + rdbEngine.enclose("pk") + + "))"; + } else if (JdbcTestUtils.isOracle(rdbEngine)) { + sql = + "CREATE TABLE " + + rdbEngine.encloseFullTableName(namespace, table) + + "(" + + rdbEngine.enclose("pk") + + " CHAR(8)," + + rdbEngine.enclose("col1") + + " CHAR(8)," + + rdbEngine.enclose("col2") + + " DATE," + + "PRIMARY KEY(" + + rdbEngine.enclose("pk") + + "))"; + } else if (JdbcTestUtils.isPostgresql(rdbEngine) || JdbcTestUtils.isSqlServer(rdbEngine)) { + sql = + "CREATE TABLE " + + rdbEngine.encloseFullTableName(namespace, table) + + "(" + + rdbEngine.enclose("pk") + + " CHAR(8)," + + rdbEngine.enclose("col1") + + " CHAR(8)," + + "PRIMARY KEY(" + + rdbEngine.enclose("pk") + + "))"; + } else if (JdbcTestUtils.isDb2(rdbEngine)) { + sql = + "CREATE TABLE " + + rdbEngine.encloseFullTableName(namespace, table) + + "(" + + rdbEngine.enclose("pk") + + " CHAR(8) NOT NULL," + + rdbEngine.enclose("col1") + + " CHAR(8)," + + rdbEngine.enclose("col2") + + " TIMESTAMP," + + "PRIMARY KEY(" + + rdbEngine.enclose("pk") + + "))"; + } else { + throw new AssertionError(); + } + + testUtils.execute(sql); + } + + @Override + protected Map getImportableTableOverrideColumnsType() { + // col1 type override confirms overriding with the default data type mapping does not fail + // col2 really performs a type override + if (JdbcTestUtils.isMysql(rdbEngine)) { + return ImmutableMap.of("col1", DataType.TEXT, "col2", DataType.TIMESTAMPTZ); + } else if (JdbcTestUtils.isOracle(rdbEngine)) { + return ImmutableMap.of("col1", DataType.TEXT, "col2", DataType.TIMESTAMP); + } else if (JdbcTestUtils.isPostgresql(rdbEngine) || JdbcTestUtils.isSqlServer(rdbEngine)) { + return ImmutableMap.of("col1", DataType.TEXT); + } else if (JdbcTestUtils.isDb2(rdbEngine)) { + return ImmutableMap.of("col1", DataType.TEXT, "col2", DataType.TIMESTAMPTZ); + } else { + throw new AssertionError(); + } + } + + @Override + protected TableMetadata getImportableTableMetadata(boolean hasTypeOverride) { + TableMetadata.Builder metadata = TableMetadata.newBuilder(); + metadata.addPartitionKey("pk"); + metadata.addColumn("pk", DataType.TEXT); + metadata.addColumn("col1", DataType.TEXT); + + if (JdbcTestUtils.isMysql(rdbEngine)) { + return metadata + .addColumn("col2", hasTypeOverride ? DataType.TIMESTAMPTZ : DataType.TIMESTAMP) + .build(); + } else if (JdbcTestUtils.isOracle(rdbEngine)) { + return metadata + .addColumn("col2", hasTypeOverride ? DataType.TIMESTAMP : DataType.DATE) + .build(); + } else if (JdbcTestUtils.isPostgresql(rdbEngine) || JdbcTestUtils.isSqlServer(rdbEngine)) { + return metadata.build(); + } else if (JdbcTestUtils.isDb2(rdbEngine)) { + return metadata + .addColumn("col2", hasTypeOverride ? DataType.TIMESTAMPTZ : DataType.TIMESTAMP) + .build(); + } else { + throw new AssertionError(); + } + } + + @SuppressFBWarnings("SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE") + @Override + protected void createNonImportableTable(String namespace, String table) throws Exception { + String nonImportableDataType; + if (JdbcTestUtils.isMysql(rdbEngine)) { + nonImportableDataType = "YEAR"; + } else if (JdbcTestUtils.isPostgresql(rdbEngine)) { + nonImportableDataType = "INTERVAL"; + } else if (JdbcTestUtils.isOracle(rdbEngine)) { + nonImportableDataType = "INT"; + } else if (JdbcTestUtils.isSqlServer(rdbEngine)) { + nonImportableDataType = "MONEY"; + } else if (JdbcTestUtils.isDb2(rdbEngine)) { + nonImportableDataType = "XML"; + } else { + throw new AssertionError(); + } + testUtils.execute( + "CREATE TABLE " + + rdbEngine.encloseFullTableName(namespace, table) + + "(" + + rdbEngine.enclose("pk") + + " CHAR(8) NOT NULL," + + rdbEngine.enclose("col") + + " " + + nonImportableDataType + + ", PRIMARY KEY(" + + rdbEngine.enclose("pk") + + "))"); + } + + @Override + protected void dropNonImportableTable(String namespace, String table) throws Exception { + testUtils.dropTable(namespace, table); + } + + @AfterAll + @Override + public void afterAll() { + try { + super.afterAll(); + } catch (Exception e) { + logger.warn("Failed to call super.afterAll", e); + } + + try { + if (testUtils != null) { + testUtils.close(); + } + } catch (Exception e) { + logger.warn("Failed to close test utils", e); + } + } + + @SuppressWarnings("unused") + private static boolean isSqliteOrOracle() { + return JdbcEnv.isSqlite() || JdbcEnv.isOracle(); + } + + @Override + protected void waitForDifferentSessionDdl() { + if (JdbcTestUtils.isYugabyte(rdbEngine)) { + // This is needed to avoid schema or catalog version mismatch database errors. + Uninterruptibles.sleepUninterruptibly(1000, TimeUnit.MILLISECONDS); + return; + } + super.waitForDifferentSessionDdl(); + } +} diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcSchemaLoaderWithMetadataDecouplingIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcSchemaLoaderWithMetadataDecouplingIntegrationTest.java new file mode 100644 index 0000000000..ad2a5747b9 --- /dev/null +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcSchemaLoaderWithMetadataDecouplingIntegrationTest.java @@ -0,0 +1,56 @@ +package com.scalar.db.storage.jdbc; + +import com.google.common.util.concurrent.Uninterruptibles; +import com.google.errorprone.annotations.concurrent.LazyInit; +import com.scalar.db.config.DatabaseConfig; +import com.scalar.db.schemaloader.SchemaLoaderWithMetadataDecouplingIntegrationTestBase; +import com.scalar.db.util.AdminTestUtils; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.condition.DisabledIf; + +@DisabledIf("com.scalar.db.storage.jdbc.JdbcEnv#isOracle") +public class JdbcSchemaLoaderWithMetadataDecouplingIntegrationTest + extends SchemaLoaderWithMetadataDecouplingIntegrationTestBase { + @LazyInit private RdbEngineStrategy rdbEngine; + + @Override + protected Properties getProperties(String testName) { + Properties properties = ConsensusCommitJdbcEnv.getProperties(testName); + JdbcConfig config = new JdbcConfig(new DatabaseConfig(properties)); + rdbEngine = RdbEngineFactory.create(config); + + // Set the isolation level for consistency reads for virtual tables + properties.setProperty( + JdbcConfig.ISOLATION_LEVEL, + JdbcTestUtils.getIsolationLevel( + rdbEngine.getMinimumIsolationLevelForConsistentVirtualTableRead()) + .name()); + + return properties; + } + + @Override + protected AdminTestUtils getAdminTestUtils(String testName) { + return new JdbcAdminTestUtils(getProperties(testName)); + } + + @Override + protected void waitForCreationIfNecessary() { + if (JdbcTestUtils.isYugabyte(rdbEngine)) { + Uninterruptibles.sleepUninterruptibly(1000, TimeUnit.MILLISECONDS); + return; + } + super.waitForCreationIfNecessary(); + } + + // Reading namespace information right after table deletion could fail with YugabyteDB. + // It should be retried. + @Override + protected boolean couldFailToReadNamespaceAfterDeletingTable() { + if (JdbcTestUtils.isYugabyte(rdbEngine)) { + return true; + } + return super.couldFailToReadNamespaceAfterDeletingTable(); + } +} diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcTestUtils.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcTestUtils.java index e3d05afd71..b12ab9d170 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcTestUtils.java +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcTestUtils.java @@ -6,6 +6,7 @@ import com.scalar.db.io.FloatColumn; import com.scalar.db.io.TextColumn; import com.scalar.db.util.TestUtils; +import java.sql.Connection; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -120,4 +121,19 @@ public static List filterDataTypes( }) .collect(Collectors.toList()); } + + public static Isolation getIsolationLevel(int isolationLevel) { + switch (isolationLevel) { + case Connection.TRANSACTION_READ_UNCOMMITTED: + return Isolation.READ_UNCOMMITTED; + case Connection.TRANSACTION_READ_COMMITTED: + return Isolation.READ_COMMITTED; + case Connection.TRANSACTION_REPEATABLE_READ: + return Isolation.REPEATABLE_READ; + case Connection.TRANSACTION_SERIALIZABLE: + return Isolation.SERIALIZABLE; + default: + throw new IllegalArgumentException("Invalid isolation level: " + isolationLevel); + } + } } diff --git a/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitNullMetadataIntegrationTestWithMultiStorage.java b/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitNullMetadataIntegrationTestWithMultiStorage.java index b87dc08ebf..490b962e85 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitNullMetadataIntegrationTestWithMultiStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitNullMetadataIntegrationTestWithMultiStorage.java @@ -42,12 +42,7 @@ protected Properties getProperties(String testName) { // the coordinator namespace to cassandra properties.setProperty( MultiStorageConfig.NAMESPACE_MAPPING, - getNamespace1() - + ":cassandra," - + getNamespace2() - + ":jdbc," - + Coordinator.NAMESPACE - + ":cassandra"); + namespace1 + ":cassandra," + namespace2 + ":jdbc," + Coordinator.NAMESPACE + ":cassandra"); // The default storage is cassandra properties.setProperty(MultiStorageConfig.DEFAULT_STORAGE, "cassandra"); diff --git a/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitSpecificIntegrationTestWithMultiStorage.java b/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitSpecificIntegrationTestWithMultiStorage.java index d24beee9e6..96b33cba08 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitSpecificIntegrationTestWithMultiStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/multistorage/ConsensusCommitSpecificIntegrationTestWithMultiStorage.java @@ -42,12 +42,7 @@ protected Properties getProperties(String testName) { // the coordinator namespace to cassandra properties.setProperty( MultiStorageConfig.NAMESPACE_MAPPING, - getNamespace1() - + ":cassandra," - + getNamespace2() - + ":jdbc," - + Coordinator.NAMESPACE - + ":cassandra"); + namespace1 + ":cassandra," + namespace2 + ":jdbc," + Coordinator.NAMESPACE + ":cassandra"); // The default storage is cassandra properties.setProperty(MultiStorageConfig.DEFAULT_STORAGE, "cassandra"); diff --git a/core/src/integration-test/java/com/scalar/db/storage/multistorage/MultiStorageSchemaLoaderIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/multistorage/MultiStorageSchemaLoaderIntegrationTest.java index 6622debabd..dcb8194453 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/multistorage/MultiStorageSchemaLoaderIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/multistorage/MultiStorageSchemaLoaderIntegrationTest.java @@ -43,12 +43,7 @@ protected Properties getProperties(String testName) { // the coordinator namespace to cassandra properties.setProperty( MultiStorageConfig.NAMESPACE_MAPPING, - getNamespace1() - + ":cassandra," - + getNamespace2() - + ":jdbc," - + Coordinator.NAMESPACE - + ":cassandra"); + namespace1 + ":cassandra," + namespace2 + ":jdbc," + Coordinator.NAMESPACE + ":cassandra"); // The default storage is cassandra properties.setProperty(MultiStorageConfig.DEFAULT_STORAGE, "cassandra"); diff --git a/core/src/main/java/com/scalar/db/common/CommonDistributedStorageAdmin.java b/core/src/main/java/com/scalar/db/common/CommonDistributedStorageAdmin.java index 27f7060c8c..b52a9126c7 100644 --- a/core/src/main/java/com/scalar/db/common/CommonDistributedStorageAdmin.java +++ b/core/src/main/java/com/scalar/db/common/CommonDistributedStorageAdmin.java @@ -489,10 +489,6 @@ public void upgrade(Map options) throws ExecutionException { @Override public StorageInfo getStorageInfo(String namespace) throws ExecutionException { - if (!namespaceExists(namespace)) { - throw new IllegalArgumentException(CoreError.NAMESPACE_NOT_FOUND.buildMessage(namespace)); - } - try { return admin.getStorageInfo(namespace); } catch (ExecutionException e) { diff --git a/core/src/main/java/com/scalar/db/common/CoreError.java b/core/src/main/java/com/scalar/db/common/CoreError.java index fd22551af1..d907f54375 100644 --- a/core/src/main/java/com/scalar/db/common/CoreError.java +++ b/core/src/main/java/com/scalar/db/common/CoreError.java @@ -1015,6 +1015,19 @@ public enum CoreError implements ScalarDbError { "The DeleteIf IS_NULL condition for right source table columns is not allowed in LEFT_OUTER virtual tables. Virtual table: %s", "", ""), + CONSENSUS_COMMIT_TRANSACTION_METADATA_DECOUPLING_NOT_SUPPORTED_STORAGE( + Category.USER_ERROR, + "0277", + "The transaction metadata decoupling feature is not supported in the storage. Storage %s", + "", + ""), + CONSENSUS_COMMIT_TRANSACTION_METADATA_CONSISTENT_READS_NOT_GUARANTEED_STORAGE( + Category.USER_ERROR, + "0278", + "The storage does not guarantee consistent reads for virtual tables. Depending on the storage configuration, " + + "you may be able to adjust the settings to enable consistent reads. Please refer to the storage configuration for details. Storage: %s", + "", + ""), // // Errors for the concurrency error category diff --git a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcUtils.java b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcUtils.java index d5cec3a5b1..a1c47a1bb9 100644 --- a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcUtils.java +++ b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcUtils.java @@ -41,25 +41,8 @@ public static BasicDataSource initDataSource( config .getIsolation() .ifPresent( - isolation -> { - switch (isolation) { - case READ_UNCOMMITTED: - dataSource.setDefaultTransactionIsolation( - Connection.TRANSACTION_READ_UNCOMMITTED); - break; - case READ_COMMITTED: - dataSource.setDefaultTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); - break; - case REPEATABLE_READ: - dataSource.setDefaultTransactionIsolation(Connection.TRANSACTION_REPEATABLE_READ); - break; - case SERIALIZABLE: - dataSource.setDefaultTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE); - break; - default: - throw new AssertionError(); - } - }); + isolation -> + dataSource.setDefaultTransactionIsolation(toJdbcTransactionIsolation(isolation))); dataSource.setDefaultReadOnly(false); @@ -90,6 +73,12 @@ public static BasicDataSource initDataSourceForTableMetadata( config.getUsername().ifPresent(dataSource::setUsername); config.getPassword().ifPresent(dataSource::setPassword); + config + .getIsolation() + .ifPresent( + isolation -> + dataSource.setDefaultTransactionIsolation(toJdbcTransactionIsolation(isolation))); + dataSource.setDefaultReadOnly(false); dataSource.setMinIdle(config.getTableMetadataConnectionPoolMinIdle()); @@ -117,6 +106,12 @@ public static BasicDataSource initDataSourceForAdmin( config.getUsername().ifPresent(dataSource::setUsername); config.getPassword().ifPresent(dataSource::setPassword); + config + .getIsolation() + .ifPresent( + isolation -> + dataSource.setDefaultTransactionIsolation(toJdbcTransactionIsolation(isolation))); + dataSource.setDefaultReadOnly(false); dataSource.setMinIdle(config.getAdminConnectionPoolMinIdle()); @@ -128,6 +123,21 @@ public static BasicDataSource initDataSourceForAdmin( return dataSource; } + private static int toJdbcTransactionIsolation(Isolation isolation) { + switch (isolation) { + case READ_UNCOMMITTED: + return Connection.TRANSACTION_READ_UNCOMMITTED; + case READ_COMMITTED: + return Connection.TRANSACTION_READ_COMMITTED; + case REPEATABLE_READ: + return Connection.TRANSACTION_REPEATABLE_READ; + case SERIALIZABLE: + return Connection.TRANSACTION_SERIALIZABLE; + default: + throw new AssertionError(); + } + } + /** * Get {@code JDBCType} of the specified {@code sqlType}. * diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdmin.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdmin.java index 5e036df4e4..a7330c41ee 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdmin.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdmin.java @@ -10,7 +10,9 @@ import com.google.inject.Inject; import com.scalar.db.api.DistributedStorageAdmin; import com.scalar.db.api.DistributedTransactionAdmin; +import com.scalar.db.api.StorageInfo; import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.VirtualTableJoinType; import com.scalar.db.common.CoreError; import com.scalar.db.config.DatabaseConfig; import com.scalar.db.exception.storage.ExecutionException; @@ -29,6 +31,13 @@ @ThreadSafe public class ConsensusCommitAdmin implements DistributedTransactionAdmin { + public static final String TRANSACTION_METADATA_DECOUPLING = "transaction-metadata-decoupling"; + + private static final String TRANSACTION_METADATA_DECOUPLING_DATA_TABLE_SUFFIX = "_data"; + private static final String TRANSACTION_METADATA_DECOUPLING_METADATA_TABLE_SUFFIX = + "_tx_metadata"; + private static final String TRANSACTION_METADATA_DECOUPLING_IMPORTED_TABLE_SUFFIX = "_scalardb"; + private final DistributedStorageAdmin admin; private final String coordinatorNamespace; private final boolean isIncludeMetadataEnabled; @@ -112,6 +121,48 @@ public void createTable( throws ExecutionException { checkNamespace(namespace); + if (isTransactionMetadataDecouplingEnabled(options)) { + // For transaction metadata decoupling mode + + throwIfTransactionMetadataDecouplingUnsupported(namespace); + + String dataTableName = table + TRANSACTION_METADATA_DECOUPLING_DATA_TABLE_SUFFIX; + String txMetadataTableName = table + TRANSACTION_METADATA_DECOUPLING_METADATA_TABLE_SUFFIX; + + if (admin.tableExists(namespace, dataTableName)) { + throw new IllegalArgumentException( + CoreError.TABLE_ALREADY_EXISTS.buildMessage( + ScalarDbUtils.getFullTableName(namespace, dataTableName))); + } + if (admin.tableExists(namespace, txMetadataTableName)) { + throw new IllegalArgumentException( + CoreError.TABLE_ALREADY_EXISTS.buildMessage( + ScalarDbUtils.getFullTableName(namespace, txMetadataTableName))); + } + + // Create a data table + admin.createTable(namespace, dataTableName, metadata, options); + + // Create a transaction metadata table + admin.createTable( + namespace, + txMetadataTableName, + ConsensusCommitUtils.buildTransactionMetadataTableMetadata(metadata), + options); + + // Create a virtual table based on the data table and the transaction metadata table + admin.createVirtualTable( + namespace, + table, + namespace, + dataTableName, + namespace, + txMetadataTableName, + VirtualTableJoinType.INNER, + options); + return; + } + admin.createTable(namespace, table, buildTransactionTableMetadata(metadata), options); } @@ -119,6 +170,35 @@ public void createTable( public void dropTable(String namespace, String table) throws ExecutionException { checkNamespace(namespace); + if (admin.getVirtualTableInfo(namespace, table).isPresent()) { + // For transaction metadata decoupling tables + + // Drop a virtual table + admin.dropTable(namespace, table); + + if (table.endsWith(TRANSACTION_METADATA_DECOUPLING_IMPORTED_TABLE_SUFFIX)) { + // For imported tables + + String originalTableName = + table.substring( + 0, table.length() - TRANSACTION_METADATA_DECOUPLING_IMPORTED_TABLE_SUFFIX.length()); + + // Drop the original data table and the transaction metadata table + admin.dropTable(namespace, originalTableName); + admin.dropTable( + namespace, originalTableName + TRANSACTION_METADATA_DECOUPLING_METADATA_TABLE_SUFFIX); + return; + } else { + // For created tables + + // Drop the data table and the transaction metadata table + admin.dropTable(namespace, table + TRANSACTION_METADATA_DECOUPLING_DATA_TABLE_SUFFIX); + admin.dropTable(namespace, table + TRANSACTION_METADATA_DECOUPLING_METADATA_TABLE_SUFFIX); + } + return; + } + + // For normal tables admin.dropTable(namespace, table); } @@ -211,6 +291,7 @@ public void repairTable( String namespace, String table, TableMetadata metadata, Map options) throws ExecutionException { checkNamespace(namespace); + throwIfTransactionMetadataDecouplingApplied(namespace, table, "repairTable()"); admin.repairTable(namespace, table, buildTransactionTableMetadata(metadata), options); } @@ -226,6 +307,7 @@ public void addNewColumnToTable( String namespace, String table, String columnName, DataType columnType) throws ExecutionException { checkNamespace(namespace); + throwIfTransactionMetadataDecouplingApplied(namespace, table, "addNewColumnToTable()"); TableMetadata tableMetadata = getTableMetadata(namespace, table); if (tableMetadata == null) { @@ -242,6 +324,7 @@ public void addNewColumnToTable( public void dropColumnFromTable(String namespace, String table, String columnName) throws ExecutionException { checkNamespace(namespace); + throwIfTransactionMetadataDecouplingApplied(namespace, table, "dropColumnFromTable()"); TableMetadata tableMetadata = getTableMetadata(namespace, table); if (tableMetadata == null) { @@ -259,6 +342,7 @@ public void renameColumn( String namespace, String table, String oldColumnName, String newColumnName) throws ExecutionException { checkNamespace(namespace); + throwIfTransactionMetadataDecouplingApplied(namespace, table, "renameColumn()"); TableMetadata tableMetadata = getTableMetadata(namespace, table); if (tableMetadata == null) { @@ -282,6 +366,7 @@ public void alterColumnType( String namespace, String table, String columnName, DataType newColumnType) throws ExecutionException { checkNamespace(namespace); + throwIfTransactionMetadataDecouplingApplied(namespace, table, "alterColumnType()"); TableMetadata tableMetadata = getTableMetadata(namespace, table); if (tableMetadata == null) { @@ -303,6 +388,7 @@ public void alterColumnType( public void renameTable(String namespace, String oldTableName, String newTableName) throws ExecutionException { checkNamespace(namespace); + throwIfTransactionMetadataDecouplingApplied(namespace, oldTableName, "renameTable()"); admin.renameTable(namespace, oldTableName, newTableName); } @@ -323,6 +409,51 @@ public void importTable( throws ExecutionException { checkNamespace(namespace); + if (isTransactionMetadataDecouplingEnabled(options)) { + // For transaction metadata decoupling mode + + throwIfTransactionMetadataDecouplingUnsupported(namespace); + + String importedTableName = table + TRANSACTION_METADATA_DECOUPLING_IMPORTED_TABLE_SUFFIX; + String txMetadataTableName = table + TRANSACTION_METADATA_DECOUPLING_METADATA_TABLE_SUFFIX; + + if (admin.tableExists(namespace, txMetadataTableName)) { + throw new IllegalArgumentException( + CoreError.TABLE_ALREADY_EXISTS.buildMessage( + ScalarDbUtils.getFullTableName(namespace, txMetadataTableName))); + } + if (admin.tableExists(namespace, importedTableName)) { + throw new IllegalArgumentException( + CoreError.TABLE_ALREADY_EXISTS.buildMessage( + ScalarDbUtils.getFullTableName(namespace, importedTableName))); + } + + // import the original table as a data table + admin.importTable(namespace, table, options, overrideColumnsType); + + TableMetadata dataTableMetadata = admin.getTableMetadata(namespace, table); + assert dataTableMetadata != null; + + // create a transaction metadata table + admin.createTable( + namespace, + txMetadataTableName, + ConsensusCommitUtils.buildTransactionMetadataTableMetadata(dataTableMetadata), + options); + + // create a virtual table based on the data table and the transaction metadata table + admin.createVirtualTable( + namespace, + importedTableName, + namespace, + table, + namespace, + txMetadataTableName, + VirtualTableJoinType.LEFT_OUTER, + options); + return; + } + // import the original table admin.importTable(namespace, table, options, overrideColumnsType); @@ -399,4 +530,36 @@ private void checkNamespace(String namespace) { CoreError.CONSENSUS_COMMIT_COORDINATOR_NAMESPACE_SPECIFIED.buildMessage(namespace)); } } + + private boolean isTransactionMetadataDecouplingEnabled(Map options) { + return options.containsKey(TRANSACTION_METADATA_DECOUPLING) + && options.get(TRANSACTION_METADATA_DECOUPLING).equalsIgnoreCase("true"); + } + + private void throwIfTransactionMetadataDecouplingUnsupported(String namespace) + throws ExecutionException { + StorageInfo storageInfo = admin.getStorageInfo(namespace); + if ((storageInfo.getMutationAtomicityUnit() != StorageInfo.MutationAtomicityUnit.STORAGE + && storageInfo.getMutationAtomicityUnit() != StorageInfo.MutationAtomicityUnit.NAMESPACE)) { + throw new IllegalArgumentException( + CoreError.CONSENSUS_COMMIT_TRANSACTION_METADATA_DECOUPLING_NOT_SUPPORTED_STORAGE + .buildMessage(storageInfo.getStorageName())); + } + if (!storageInfo.isConsistentVirtualTableReadGuaranteed()) { + throw new IllegalArgumentException( + CoreError.CONSENSUS_COMMIT_TRANSACTION_METADATA_CONSISTENT_READS_NOT_GUARANTEED_STORAGE + .buildMessage(storageInfo.getStorageName())); + } + } + + private void throwIfTransactionMetadataDecouplingApplied( + String namespace, String table, String method) throws ExecutionException { + if (admin.tableExists(namespace, table) + && admin.getVirtualTableInfo(namespace, table).isPresent()) { + throw new UnsupportedOperationException( + "Currently, " + + method + + " is not supported for tables that applies transaction metadata decoupling"); + } + } } diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java index c04facb622..c73159ebcd 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java @@ -26,6 +26,7 @@ import com.scalar.db.common.AbstractTransactionManagerCrudOperableScanner; import com.scalar.db.common.ReadOnlyDistributedTransaction; import com.scalar.db.common.StorageInfoProvider; +import com.scalar.db.common.VirtualTableInfoManager; import com.scalar.db.config.DatabaseConfig; import com.scalar.db.exception.transaction.CommitConflictException; import com.scalar.db.exception.transaction.CrudConflictException; @@ -84,11 +85,17 @@ public ConsensusCommitManager( tableMetadataManager, config.isIncludeMetadataEnabled(), parallelExecutor); - commit = createCommitHandler(config); + StorageInfoProvider storageInfoProvider = new StorageInfoProvider(admin); + commit = createCommitHandler(config, storageInfoProvider); isolation = config.getIsolation(); + VirtualTableInfoManager virtualTableInfoManager = + new VirtualTableInfoManager(admin, databaseConfig.getMetadataCacheExpirationTimeSecs()); operationChecker = new ConsensusCommitOperationChecker( - tableMetadataManager, config.isIncludeMetadataEnabled()); + tableMetadataManager, + virtualTableInfoManager, + storageInfoProvider, + config.isIncludeMetadataEnabled()); } protected ConsensusCommitManager(DatabaseConfig databaseConfig) { @@ -113,11 +120,17 @@ protected ConsensusCommitManager(DatabaseConfig databaseConfig) { tableMetadataManager, config.isIncludeMetadataEnabled(), parallelExecutor); - commit = createCommitHandler(config); + StorageInfoProvider storageInfoProvider = new StorageInfoProvider(admin); + commit = createCommitHandler(config, storageInfoProvider); isolation = config.getIsolation(); + VirtualTableInfoManager virtualTableInfoManager = + new VirtualTableInfoManager(admin, databaseConfig.getMetadataCacheExpirationTimeSecs()); operationChecker = new ConsensusCommitOperationChecker( - tableMetadataManager, config.isIncludeMetadataEnabled()); + tableMetadataManager, + virtualTableInfoManager, + storageInfoProvider, + config.isIncludeMetadataEnabled()); } @SuppressFBWarnings("EI_EXPOSE_REP2") @@ -147,14 +160,21 @@ protected ConsensusCommitManager(DatabaseConfig databaseConfig) { this.commit = commit; this.groupCommitter = groupCommitter; this.isolation = isolation; + VirtualTableInfoManager virtualTableInfoManager = + new VirtualTableInfoManager(admin, databaseConfig.getMetadataCacheExpirationTimeSecs()); + StorageInfoProvider storageInfoProvider = new StorageInfoProvider(admin); this.operationChecker = new ConsensusCommitOperationChecker( - tableMetadataManager, config.isIncludeMetadataEnabled()); + tableMetadataManager, + virtualTableInfoManager, + storageInfoProvider, + config.isIncludeMetadataEnabled()); } // `groupCommitter` must be set before calling this method. - private CommitHandler createCommitHandler(ConsensusCommitConfig config) { - MutationsGrouper mutationsGrouper = new MutationsGrouper(new StorageInfoProvider(admin)); + private CommitHandler createCommitHandler( + ConsensusCommitConfig config, StorageInfoProvider storageInfoProvider) { + MutationsGrouper mutationsGrouper = new MutationsGrouper(storageInfoProvider); if (isGroupCommitEnabled()) { return new CommitHandlerWithGroupCommit( storage, diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitOperationChecker.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitOperationChecker.java index 6076fe201c..ba03c7644a 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitOperationChecker.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitOperationChecker.java @@ -10,6 +10,7 @@ import com.scalar.db.api.Get; import com.scalar.db.api.Mutation; import com.scalar.db.api.MutationCondition; +import com.scalar.db.api.Operation; import com.scalar.db.api.Put; import com.scalar.db.api.PutIf; import com.scalar.db.api.PutIfExists; @@ -17,8 +18,12 @@ import com.scalar.db.api.Scan; import com.scalar.db.api.ScanAll; import com.scalar.db.api.Selection; +import com.scalar.db.api.StorageInfo; import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.VirtualTableInfo; import com.scalar.db.common.CoreError; +import com.scalar.db.common.StorageInfoProvider; +import com.scalar.db.common.VirtualTableInfoManager; import com.scalar.db.common.checker.ConditionChecker; import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.util.ScalarDbUtils; @@ -28,12 +33,18 @@ public class ConsensusCommitOperationChecker { private final TransactionTableMetadataManager transactionTableMetadataManager; + private final VirtualTableInfoManager virtualTableInfoManager; + private final StorageInfoProvider storageInfoProvider; private final boolean isIncludeMetadataEnabled; public ConsensusCommitOperationChecker( TransactionTableMetadataManager transactionTableMetadataManager, + VirtualTableInfoManager virtualTableInfoManager, + StorageInfoProvider storageInfoProvider, boolean isIncludeMetadataEnabled) { this.transactionTableMetadataManager = transactionTableMetadataManager; + this.virtualTableInfoManager = virtualTableInfoManager; + this.storageInfoProvider = storageInfoProvider; this.isIncludeMetadataEnabled = isIncludeMetadataEnabled; } @@ -46,6 +57,8 @@ public ConsensusCommitOperationChecker( * @throws IllegalArgumentException when the get is invalid */ public void check(Get get, TransactionContext context) throws ExecutionException { + throwIfOperationForVirtualTableButNotConsistentVirtualTableReadStorage(get); + TransactionTableMetadata metadata = getTransactionTableMetadata(transactionTableMetadataManager, get); @@ -100,6 +113,8 @@ public void check(Get get, TransactionContext context) throws ExecutionException * @throws IllegalArgumentException when the scan is invalid */ public void check(Scan scan, TransactionContext context) throws ExecutionException { + throwIfOperationForVirtualTableButNotConsistentVirtualTableReadStorage(scan); + TransactionTableMetadata metadata = getTransactionTableMetadata(transactionTableMetadataManager, scan); @@ -182,6 +197,8 @@ public void check(Scan scan, TransactionContext context) throws ExecutionExcepti * @throws IllegalArgumentException when the mutation is invalid */ public void check(Mutation mutation) throws ExecutionException { + throwIfOperationForVirtualTableButNotConsistentVirtualTableReadStorage(mutation); + if (mutation instanceof Put) { check((Put) mutation); } else { @@ -253,4 +270,21 @@ private void checkConditionIsNotTargetingMetadataColumns( ConditionChecker createConditionChecker(TableMetadata tableMetadata) { return new ConditionChecker(tableMetadata); } + + private void throwIfOperationForVirtualTableButNotConsistentVirtualTableReadStorage( + Operation operation) throws ExecutionException { + assert operation.forNamespace().isPresent(); + + VirtualTableInfo virtualTableInfo = virtualTableInfoManager.getVirtualTableInfo(operation); + if (virtualTableInfo == null) { + return; + } + + StorageInfo storageInfo = storageInfoProvider.getStorageInfo(operation.forNamespace().get()); + if (!storageInfo.isConsistentVirtualTableReadGuaranteed()) { + throw new IllegalArgumentException( + CoreError.CONSENSUS_COMMIT_TRANSACTION_METADATA_CONSISTENT_READS_NOT_GUARANTEED_STORAGE + .buildMessage(storageInfo.getStorageName())); + } + } } diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtils.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtils.java index f83d627c2d..6e873b3e5c 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtils.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtils.java @@ -83,6 +83,41 @@ public static TableMetadata buildTransactionTableMetadata(TableMetadata tableMet return builder.build(); } + /** + * Build a table metadata of a decoupled transaction metadata table based on the specified table + * metadata. This is for the transaction metadata decoupling feature. + * + * @param tableMetadata the table metadata of the data table + * @return the table metadata of a transaction metadata table + */ + public static TableMetadata buildTransactionMetadataTableMetadata(TableMetadata tableMetadata) { + checkIsNotTransactionMetaColumn(tableMetadata.getColumnNames()); + Set nonPrimaryKeyColumns = getNonPrimaryKeyColumns(tableMetadata); + checkBeforeColumnsDoNotAlreadyExist(nonPrimaryKeyColumns, tableMetadata); + + TableMetadata.Builder builder = TableMetadata.newBuilder(); + + // Add primary key columns + for (String partitionKeyName : tableMetadata.getPartitionKeyNames()) { + builder.addColumn(partitionKeyName, tableMetadata.getColumnDataType(partitionKeyName)); + builder.addPartitionKey(partitionKeyName); + } + for (String clusteringKeyName : tableMetadata.getClusteringKeyNames()) { + builder.addColumn(clusteringKeyName, tableMetadata.getColumnDataType(clusteringKeyName)); + builder.addClusteringKey( + clusteringKeyName, tableMetadata.getClusteringOrder(clusteringKeyName)); + } + + // Add transaction meta columns + TRANSACTION_META_COLUMNS.forEach(builder::addColumn); + + // Add before image columns for non-primary key columns + nonPrimaryKeyColumns.forEach( + c -> builder.addColumn(Attribute.BEFORE_PREFIX + c, tableMetadata.getColumnDataType(c))); + + return builder.build(); + } + private static void checkIsNotTransactionMetaColumn(Set columnNames) { TRANSACTION_META_COLUMNS .keySet() diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java index 9f6058258b..a4f48c3a66 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java @@ -24,6 +24,7 @@ import com.scalar.db.common.AbstractTwoPhaseCommitTransactionManager; import com.scalar.db.common.CoreError; import com.scalar.db.common.StorageInfoProvider; +import com.scalar.db.common.VirtualTableInfoManager; import com.scalar.db.config.DatabaseConfig; import com.scalar.db.exception.transaction.CommitConflictException; import com.scalar.db.exception.transaction.CrudConflictException; @@ -85,18 +86,24 @@ public TwoPhaseConsensusCommitManager( tableMetadataManager, config.isIncludeMetadataEnabled(), parallelExecutor); + StorageInfoProvider storageInfoProvider = new StorageInfoProvider(admin); commit = new CommitHandler( storage, coordinator, tableMetadataManager, parallelExecutor, - new MutationsGrouper(new StorageInfoProvider(admin)), + new MutationsGrouper(storageInfoProvider), config.isCoordinatorWriteOmissionOnReadOnlyEnabled(), config.isOnePhaseCommitEnabled()); + VirtualTableInfoManager virtualTableInfoManager = + new VirtualTableInfoManager(admin, databaseConfig.getMetadataCacheExpirationTimeSecs()); operationChecker = new ConsensusCommitOperationChecker( - tableMetadataManager, config.isIncludeMetadataEnabled()); + tableMetadataManager, + virtualTableInfoManager, + storageInfoProvider, + config.isIncludeMetadataEnabled()); } public TwoPhaseConsensusCommitManager(DatabaseConfig databaseConfig) { @@ -119,18 +126,24 @@ public TwoPhaseConsensusCommitManager(DatabaseConfig databaseConfig) { tableMetadataManager, config.isIncludeMetadataEnabled(), parallelExecutor); + StorageInfoProvider storageInfoProvider = new StorageInfoProvider(admin); commit = new CommitHandler( storage, coordinator, tableMetadataManager, parallelExecutor, - new MutationsGrouper(new StorageInfoProvider(admin)), + new MutationsGrouper(storageInfoProvider), config.isCoordinatorWriteOmissionOnReadOnlyEnabled(), config.isOnePhaseCommitEnabled()); + VirtualTableInfoManager virtualTableInfoManager = + new VirtualTableInfoManager(admin, databaseConfig.getMetadataCacheExpirationTimeSecs()); operationChecker = new ConsensusCommitOperationChecker( - tableMetadataManager, config.isIncludeMetadataEnabled()); + tableMetadataManager, + virtualTableInfoManager, + storageInfoProvider, + config.isIncludeMetadataEnabled()); } @SuppressFBWarnings("EI_EXPOSE_REP2") @@ -157,9 +170,15 @@ public TwoPhaseConsensusCommitManager(DatabaseConfig databaseConfig) { this.recoveryExecutor = recoveryExecutor; this.crud = crud; this.commit = commit; + StorageInfoProvider storageInfoProvider = new StorageInfoProvider(admin); + VirtualTableInfoManager virtualTableInfoManager = + new VirtualTableInfoManager(admin, databaseConfig.getMetadataCacheExpirationTimeSecs()); operationChecker = new ConsensusCommitOperationChecker( - tableMetadataManager, config.isIncludeMetadataEnabled()); + tableMetadataManager, + virtualTableInfoManager, + storageInfoProvider, + config.isIncludeMetadataEnabled()); } private void throwIfGroupCommitIsEnabled() { diff --git a/core/src/test/java/com/scalar/db/storage/jdbc/JdbcUtilsTest.java b/core/src/test/java/com/scalar/db/storage/jdbc/JdbcUtilsTest.java index b8f6fb061c..5ed3f7ad27 100644 --- a/core/src/test/java/com/scalar/db/storage/jdbc/JdbcUtilsTest.java +++ b/core/src/test/java/com/scalar/db/storage/jdbc/JdbcUtilsTest.java @@ -163,6 +163,7 @@ public void initDataSourceForTableMetadata_ShouldReturnProperDataSource() throws properties.setProperty(DatabaseConfig.USERNAME, "user"); properties.setProperty(DatabaseConfig.PASSWORD, "oracle"); properties.setProperty(DatabaseConfig.STORAGE, "jdbc"); + properties.setProperty(JdbcConfig.ISOLATION_LEVEL, "REPEATABLE_READ"); properties.setProperty(JdbcConfig.TABLE_METADATA_CONNECTION_POOL_MIN_IDLE, "100"); properties.setProperty(JdbcConfig.TABLE_METADATA_CONNECTION_POOL_MAX_IDLE, "200"); properties.setProperty(JdbcConfig.TABLE_METADATA_CONNECTION_POOL_MAX_TOTAL, "300"); @@ -182,6 +183,8 @@ public void initDataSourceForTableMetadata_ShouldReturnProperDataSource() throws assertThat(tableMetadataDataSource.getUsername()).isEqualTo("user"); assertThat(tableMetadataDataSource.getPassword()).isEqualTo("oracle"); + assertThat(tableMetadataDataSource.getDefaultTransactionIsolation()) + .isEqualTo(Connection.TRANSACTION_REPEATABLE_READ); assertThat(tableMetadataDataSource.getDefaultReadOnly()).isFalse(); assertThat(tableMetadataDataSource.getMinIdle()).isEqualTo(100); @@ -199,6 +202,7 @@ public void initDataSourceForAdmin_ShouldReturnProperDataSource() throws SQLExce properties.setProperty(DatabaseConfig.USERNAME, "user"); properties.setProperty(DatabaseConfig.PASSWORD, "sqlserver"); properties.setProperty(DatabaseConfig.STORAGE, "jdbc"); + properties.setProperty(JdbcConfig.ISOLATION_LEVEL, "READ_UNCOMMITTED"); properties.setProperty(JdbcConfig.ADMIN_CONNECTION_POOL_MIN_IDLE, "100"); properties.setProperty(JdbcConfig.ADMIN_CONNECTION_POOL_MAX_IDLE, "200"); properties.setProperty(JdbcConfig.ADMIN_CONNECTION_POOL_MAX_TOTAL, "300"); @@ -216,6 +220,8 @@ public void initDataSourceForAdmin_ShouldReturnProperDataSource() throws SQLExce assertThat(adminDataSource.getUsername()).isEqualTo("user"); assertThat(adminDataSource.getPassword()).isEqualTo("sqlserver"); + assertThat(adminDataSource.getDefaultTransactionIsolation()) + .isEqualTo(Connection.TRANSACTION_READ_UNCOMMITTED); assertThat(adminDataSource.getDefaultReadOnly()).isFalse(); assertThat(adminDataSource.getMinIdle()).isEqualTo(100); diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminTestBase.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminTestBase.java index 9c09dca672..d626511e05 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminTestBase.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminTestBase.java @@ -14,7 +14,12 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.scalar.db.api.DistributedStorageAdmin; +import com.scalar.db.api.StorageInfo; +import com.scalar.db.api.StorageInfo.MutationAtomicityUnit; import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.VirtualTableInfo; +import com.scalar.db.api.VirtualTableJoinType; +import com.scalar.db.common.StorageInfoImpl; import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.io.DataType; import java.util.Collections; @@ -822,4 +827,405 @@ public void coordinatorNamespaceShouldHandleCorrectly() throws ExecutionExceptio () -> admin.importTable(coordinatorNamespaceName, "tbl", Collections.emptyMap())) .isInstanceOf(IllegalArgumentException.class); } + + @Test + public void + createTable_WithTransactionMetadataDecouplingEnabled_ShouldCreateDataTableAndMetadataTableAndVirtualTable() + throws ExecutionException { + // Arrange + final String ACCOUNT_ID = "account_id"; + final String ACCOUNT_TYPE = "account_type"; + final String BALANCE = "balance"; + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(ACCOUNT_ID, DataType.INT) + .addColumn(ACCOUNT_TYPE, DataType.INT) + .addColumn(BALANCE, DataType.INT) + .addPartitionKey(ACCOUNT_ID) + .addClusteringKey(ACCOUNT_TYPE) + .build(); + + TableMetadata expectedTxMetadataTableMetadata = + TableMetadata.newBuilder() + .addColumn(ACCOUNT_ID, DataType.INT) + .addColumn(ACCOUNT_TYPE, DataType.INT) + .addColumn(Attribute.ID, DataType.TEXT) + .addColumn(Attribute.STATE, DataType.INT) + .addColumn(Attribute.VERSION, DataType.INT) + .addColumn(Attribute.PREPARED_AT, DataType.BIGINT) + .addColumn(Attribute.COMMITTED_AT, DataType.BIGINT) + .addColumn(Attribute.BEFORE_PREFIX + BALANCE, DataType.INT) + .addColumn(Attribute.BEFORE_ID, DataType.TEXT) + .addColumn(Attribute.BEFORE_STATE, DataType.INT) + .addColumn(Attribute.BEFORE_VERSION, DataType.INT) + .addColumn(Attribute.BEFORE_PREPARED_AT, DataType.BIGINT) + .addColumn(Attribute.BEFORE_COMMITTED_AT, DataType.BIGINT) + .addPartitionKey(ACCOUNT_ID) + .addClusteringKey(ACCOUNT_TYPE) + .build(); + + Map options = + ImmutableMap.of(ConsensusCommitAdmin.TRANSACTION_METADATA_DECOUPLING, "true"); + + StorageInfo storageInfo = + new StorageInfoImpl("jdbc", MutationAtomicityUnit.STORAGE, Integer.MAX_VALUE, true); + when(distributedStorageAdmin.getStorageInfo(NAMESPACE)).thenReturn(storageInfo); + + // Act + admin.createTable(NAMESPACE, TABLE, tableMetadata, options); + + // Assert + verify(distributedStorageAdmin).getStorageInfo(NAMESPACE); + verify(distributedStorageAdmin).createTable(NAMESPACE, TABLE + "_data", tableMetadata, options); + verify(distributedStorageAdmin) + .createTable(NAMESPACE, TABLE + "_tx_metadata", expectedTxMetadataTableMetadata, options); + verify(distributedStorageAdmin) + .createVirtualTable( + NAMESPACE, + TABLE, + NAMESPACE, + TABLE + "_data", + NAMESPACE, + TABLE + "_tx_metadata", + VirtualTableJoinType.INNER, + options); + } + + @Test + public void + createTable_WithTransactionMetadataDecouplingEnabledAndUnsupportedStorage_ShouldThrowIllegalArgumentException() + throws ExecutionException { + // Arrange + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn("col1", DataType.INT) + .addColumn("col2", DataType.INT) + .addPartitionKey("col1") + .build(); + + Map options = + ImmutableMap.of(ConsensusCommitAdmin.TRANSACTION_METADATA_DECOUPLING, "true"); + + // Unsupported atomicity unit (RECORD) + StorageInfo storageInfo = + new StorageInfoImpl("cassandra", MutationAtomicityUnit.PARTITION, Integer.MAX_VALUE, true); + when(distributedStorageAdmin.getStorageInfo(NAMESPACE)).thenReturn(storageInfo); + + // Act Assert + assertThatThrownBy(() -> admin.createTable(NAMESPACE, TABLE, tableMetadata, options)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void + createTable_WithTransactionMetadataDecouplingEnabledAndConsistentReadsNotGuaranteed_ShouldThrowIllegalArgumentException() + throws ExecutionException { + // Arrange + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn("col1", DataType.INT) + .addColumn("col2", DataType.INT) + .addPartitionKey("col1") + .build(); + + Map options = + ImmutableMap.of(ConsensusCommitAdmin.TRANSACTION_METADATA_DECOUPLING, "true"); + + StorageInfo storageInfo = + new StorageInfoImpl("dynamodb", MutationAtomicityUnit.STORAGE, 100, false); + when(distributedStorageAdmin.getStorageInfo(NAMESPACE)).thenReturn(storageInfo); + + // Act Assert + assertThatThrownBy(() -> admin.createTable(NAMESPACE, TABLE, tableMetadata, options)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void + createTable_WithTransactionMetadataDecouplingEnabledAndDataTableAlreadyExists_ShouldThrowIllegalArgumentException() + throws ExecutionException { + // Arrange + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn("col1", DataType.INT) + .addColumn("col2", DataType.INT) + .addPartitionKey("col1") + .build(); + + Map options = + ImmutableMap.of(ConsensusCommitAdmin.TRANSACTION_METADATA_DECOUPLING, "true"); + + StorageInfo storageInfo = + new StorageInfoImpl("jdbc", MutationAtomicityUnit.STORAGE, Integer.MAX_VALUE, true); + when(distributedStorageAdmin.getStorageInfo(NAMESPACE)).thenReturn(storageInfo); + when(distributedStorageAdmin.tableExists(NAMESPACE, TABLE + "_data")).thenReturn(true); + + // Act Assert + assertThatThrownBy(() -> admin.createTable(NAMESPACE, TABLE, tableMetadata, options)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void + createTable_WithTransactionMetadataDecouplingEnabledAndMetadataTableAlreadyExists_ShouldThrowIllegalArgumentException() + throws ExecutionException { + // Arrange + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn("col1", DataType.INT) + .addColumn("col2", DataType.INT) + .addPartitionKey("col1") + .build(); + + Map options = + ImmutableMap.of(ConsensusCommitAdmin.TRANSACTION_METADATA_DECOUPLING, "true"); + + StorageInfo storageInfo = + new StorageInfoImpl("jdbc", MutationAtomicityUnit.STORAGE, Integer.MAX_VALUE, true); + when(distributedStorageAdmin.getStorageInfo(NAMESPACE)).thenReturn(storageInfo); + when(distributedStorageAdmin.tableExists(NAMESPACE, TABLE + "_data")).thenReturn(false); + when(distributedStorageAdmin.tableExists(NAMESPACE, TABLE + "_tx_metadata")).thenReturn(true); + + // Act Assert + assertThatThrownBy(() -> admin.createTable(NAMESPACE, TABLE, tableMetadata, options)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void dropTable_ForVirtualTable_ShouldDropVirtualTableAndSourceTables() + throws ExecutionException { + // Arrange + VirtualTableInfo virtualTableInfo = + createVirtualTableInfo( + NAMESPACE, + TABLE, + NAMESPACE, + TABLE + "_data", + NAMESPACE, + TABLE + "_tx_metadata", + VirtualTableJoinType.INNER); + when(distributedStorageAdmin.getVirtualTableInfo(NAMESPACE, TABLE)) + .thenReturn(Optional.of(virtualTableInfo)); + + // Act + admin.dropTable(NAMESPACE, TABLE); + + // Assert + verify(distributedStorageAdmin).getVirtualTableInfo(NAMESPACE, TABLE); + verify(distributedStorageAdmin).dropTable(NAMESPACE, TABLE); + verify(distributedStorageAdmin).dropTable(NAMESPACE, TABLE + "_data"); + verify(distributedStorageAdmin).dropTable(NAMESPACE, TABLE + "_tx_metadata"); + } + + @Test + public void dropTable_ForImportedVirtualTable_ShouldDropVirtualTableAndSourceTables() + throws ExecutionException { + // Arrange + String importedTableName = TABLE + "_scalardb"; + VirtualTableInfo virtualTableInfo = + createVirtualTableInfo( + NAMESPACE, + importedTableName, + NAMESPACE, + TABLE, + NAMESPACE, + TABLE + "_tx_metadata", + VirtualTableJoinType.LEFT_OUTER); + when(distributedStorageAdmin.getVirtualTableInfo(NAMESPACE, importedTableName)) + .thenReturn(Optional.of(virtualTableInfo)); + + // Act + admin.dropTable(NAMESPACE, importedTableName); + + // Assert + verify(distributedStorageAdmin).getVirtualTableInfo(NAMESPACE, importedTableName); + verify(distributedStorageAdmin).dropTable(NAMESPACE, importedTableName); + verify(distributedStorageAdmin).dropTable(NAMESPACE, TABLE); + verify(distributedStorageAdmin).dropTable(NAMESPACE, TABLE + "_tx_metadata"); + } + + private VirtualTableInfo createVirtualTableInfo( + String namespace, + String table, + String leftSourceNamespace, + String leftSourceTable, + String rightSourceNamespace, + String rightSourceTable, + VirtualTableJoinType joinType) { + return new VirtualTableInfo() { + @Override + public String getNamespaceName() { + return namespace; + } + + @Override + public String getTableName() { + return table; + } + + @Override + public String getLeftSourceNamespaceName() { + return leftSourceNamespace; + } + + @Override + public String getLeftSourceTableName() { + return leftSourceTable; + } + + @Override + public String getRightSourceNamespaceName() { + return rightSourceNamespace; + } + + @Override + public String getRightSourceTableName() { + return rightSourceTable; + } + + @Override + public VirtualTableJoinType getJoinType() { + return joinType; + } + }; + } + + @Test + public void + importTable_WithTransactionMetadataDecouplingEnabled_ShouldImportAndCreateMetadataTableAndVirtualTable() + throws ExecutionException { + // Arrange + String primaryKeyColumn = "pk"; + String column = "col"; + TableMetadata dataTableMetadata = + TableMetadata.newBuilder() + .addColumn(primaryKeyColumn, DataType.INT) + .addColumn(column, DataType.INT) + .addPartitionKey(primaryKeyColumn) + .build(); + + TableMetadata expectedTxMetadataTableMetadata = + TableMetadata.newBuilder() + .addColumn(primaryKeyColumn, DataType.INT) + .addColumn(Attribute.ID, DataType.TEXT) + .addColumn(Attribute.STATE, DataType.INT) + .addColumn(Attribute.VERSION, DataType.INT) + .addColumn(Attribute.PREPARED_AT, DataType.BIGINT) + .addColumn(Attribute.COMMITTED_AT, DataType.BIGINT) + .addColumn(Attribute.BEFORE_PREFIX + column, DataType.INT) + .addColumn(Attribute.BEFORE_ID, DataType.TEXT) + .addColumn(Attribute.BEFORE_STATE, DataType.INT) + .addColumn(Attribute.BEFORE_VERSION, DataType.INT) + .addColumn(Attribute.BEFORE_PREPARED_AT, DataType.BIGINT) + .addColumn(Attribute.BEFORE_COMMITTED_AT, DataType.BIGINT) + .addPartitionKey(primaryKeyColumn) + .build(); + + Map options = + ImmutableMap.of(ConsensusCommitAdmin.TRANSACTION_METADATA_DECOUPLING, "true"); + Map overrideColumnsType = Collections.emptyMap(); + + StorageInfo storageInfo = + new StorageInfoImpl("jdbc", MutationAtomicityUnit.STORAGE, Integer.MAX_VALUE, true); + + when(distributedStorageAdmin.getStorageInfo(NAMESPACE)).thenReturn(storageInfo); + when(distributedStorageAdmin.getTableMetadata(NAMESPACE, TABLE)).thenReturn(dataTableMetadata); + + // Act + admin.importTable(NAMESPACE, TABLE, options, overrideColumnsType); + + // Assert + verify(distributedStorageAdmin).getStorageInfo(NAMESPACE); + verify(distributedStorageAdmin).importTable(NAMESPACE, TABLE, options, overrideColumnsType); + verify(distributedStorageAdmin).getTableMetadata(NAMESPACE, TABLE); + verify(distributedStorageAdmin) + .createTable(NAMESPACE, TABLE + "_tx_metadata", expectedTxMetadataTableMetadata, options); + verify(distributedStorageAdmin) + .createVirtualTable( + NAMESPACE, + TABLE + "_scalardb", + NAMESPACE, + TABLE, + NAMESPACE, + TABLE + "_tx_metadata", + VirtualTableJoinType.LEFT_OUTER, + options); + } + + @Test + public void + importTable_WithTransactionMetadataDecouplingEnabledAndUnsupportedStorage_ShouldThrowIllegalArgumentException() + throws ExecutionException { + // Arrange + Map options = + ImmutableMap.of(ConsensusCommitAdmin.TRANSACTION_METADATA_DECOUPLING, "true"); + Map overrideColumnsType = Collections.emptyMap(); + + // Unsupported atomicity unit (PARTITION) + StorageInfo storageInfo = + new StorageInfoImpl("cassandra", MutationAtomicityUnit.PARTITION, Integer.MAX_VALUE, true); + when(distributedStorageAdmin.getStorageInfo(NAMESPACE)).thenReturn(storageInfo); + + // Act Assert + assertThatThrownBy(() -> admin.importTable(NAMESPACE, TABLE, options, overrideColumnsType)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void + importTable_WithTransactionMetadataDecouplingEnabledAndConsistentReadsNotGuaranteed_ShouldThrowIllegalArgumentException() + throws ExecutionException { + // Arrange + Map options = + ImmutableMap.of(ConsensusCommitAdmin.TRANSACTION_METADATA_DECOUPLING, "true"); + Map overrideColumnsType = Collections.emptyMap(); + + StorageInfo storageInfo = + new StorageInfoImpl("dynamodb", MutationAtomicityUnit.STORAGE, 100, false); + when(distributedStorageAdmin.getStorageInfo(NAMESPACE)).thenReturn(storageInfo); + + // Act Assert + assertThatThrownBy(() -> admin.importTable(NAMESPACE, TABLE, options, overrideColumnsType)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void + importTable_WithTransactionMetadataDecouplingEnabledAndMetadataTableAlreadyExists_ShouldThrowIllegalArgumentException() + throws ExecutionException { + // Arrange + Map options = + ImmutableMap.of(ConsensusCommitAdmin.TRANSACTION_METADATA_DECOUPLING, "true"); + Map overrideColumnsType = Collections.emptyMap(); + + StorageInfo storageInfo = + new StorageInfoImpl("jdbc", MutationAtomicityUnit.STORAGE, Integer.MAX_VALUE, true); + when(distributedStorageAdmin.getStorageInfo(NAMESPACE)).thenReturn(storageInfo); + when(distributedStorageAdmin.tableExists(NAMESPACE, TABLE + "_tx_metadata")).thenReturn(true); + + // Act Assert + assertThatThrownBy(() -> admin.importTable(NAMESPACE, TABLE, options, overrideColumnsType)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void + importTable_WithTransactionMetadataDecouplingEnabledAndImportedTableAlreadyExists_ShouldThrowIllegalArgumentException() + throws ExecutionException { + // Arrange + Map options = + ImmutableMap.of(ConsensusCommitAdmin.TRANSACTION_METADATA_DECOUPLING, "true"); + Map overrideColumnsType = Collections.emptyMap(); + + StorageInfo storageInfo = + new StorageInfoImpl("jdbc", MutationAtomicityUnit.STORAGE, Integer.MAX_VALUE, true); + when(distributedStorageAdmin.getStorageInfo(NAMESPACE)).thenReturn(storageInfo); + when(distributedStorageAdmin.tableExists(NAMESPACE, TABLE + "_tx_metadata")).thenReturn(false); + when(distributedStorageAdmin.tableExists(NAMESPACE, TABLE + "_scalardb")).thenReturn(true); + + // Act Assert + assertThatThrownBy(() -> admin.importTable(NAMESPACE, TABLE, options, overrideColumnsType)) + .isInstanceOf(IllegalArgumentException.class); + } } diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitOperationCheckerTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitOperationCheckerTest.java index 85a176aa9a..0233141dd7 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitOperationCheckerTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitOperationCheckerTest.java @@ -8,7 +8,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.google.common.collect.ImmutableSet; import com.scalar.db.api.ConditionBuilder; import com.scalar.db.api.Delete; import com.scalar.db.api.DeleteIf; @@ -21,14 +20,19 @@ import com.scalar.db.api.PutIfNotExists; import com.scalar.db.api.Scan; import com.scalar.db.api.ScanAll; +import com.scalar.db.api.StorageInfo; +import com.scalar.db.api.StorageInfo.MutationAtomicityUnit; import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.VirtualTableInfo; +import com.scalar.db.common.StorageInfoImpl; +import com.scalar.db.common.StorageInfoProvider; +import com.scalar.db.common.VirtualTableInfoManager; import com.scalar.db.common.checker.ConditionChecker; import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.io.DataType; import com.scalar.db.io.Key; import java.util.Collections; import java.util.LinkedHashSet; -import java.util.Optional; import java.util.Set; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -43,8 +47,8 @@ public class ConsensusCommitOperationCheckerTest { private static final String ANY_METADATA_COL_1 = "any_metadata_col_1"; private static final String ANY_METADATA_COL_2 = "any_metadata_col_2"; @Mock private TransactionTableMetadataManager metadataManager; - @Mock private Put put; - @Mock private Delete delete; + @Mock private VirtualTableInfoManager virtualTableInfoManager; + @Mock private StorageInfoProvider storageInfoProvider; @Mock private TransactionTableMetadata tableMetadata; @Mock private ConditionChecker conditionChecker; private ConsensusCommitOperationChecker checker; @@ -52,7 +56,10 @@ public class ConsensusCommitOperationCheckerTest { @BeforeEach public void setUp() throws Exception { MockitoAnnotations.openMocks(this).close(); - checker = spy(new ConsensusCommitOperationChecker(metadataManager, false)); + checker = + spy( + new ConsensusCommitOperationChecker( + metadataManager, virtualTableInfoManager, storageInfoProvider, false)); when(checker.createConditionChecker(any())).thenReturn(conditionChecker); when(metadataManager.getTransactionTableMetadata(any())).thenReturn(tableMetadata); LinkedHashSet metadataColumns = new LinkedHashSet<>(); @@ -66,7 +73,13 @@ public void setUp() throws Exception { public void checkForPut_WithNonAllowedCondition_ShouldThrowIllegalArgumentException( Class deleteConditonClass) { // Arrange - when(put.getCondition()).thenReturn(Optional.of(mock(deleteConditonClass))); + Put put = + Put.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofInt("pk", 1)) + .condition(mock(deleteConditonClass)) + .build(); // Act Assert assertThatThrownBy(() -> checker.check(put)).isInstanceOf(IllegalArgumentException.class); @@ -78,7 +91,13 @@ public void checkForPut_WithAllowedCondition_ShouldCallConditionChecker( Class putConditionClass) { // Arrange MutationCondition condition = mock(putConditionClass); - when(put.getCondition()).thenReturn(Optional.of(condition)); + Put put = + Put.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofInt("pk", 1)) + .condition(condition) + .build(); // Act Assert assertThatCode(() -> checker.check(put)).doesNotThrowAnyException(); @@ -90,9 +109,15 @@ public void checkForPut_ThatMutatesMetadataColumns_ShouldThrowIllegalArgumentExc throws ExecutionException { // Arrange String fullTableName = "ns.tbl"; - Set columns = ImmutableSet.of(ANY_COL_1, ANY_METADATA_COL_1, ANY_COL_2); - when(put.forFullTableName()).thenReturn(Optional.of(fullTableName)); - when(put.getContainedColumnNames()).thenReturn(columns); + Put put = + Put.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofInt("pk", 1)) + .intValue(ANY_COL_1, 1) + .textValue(ANY_METADATA_COL_1, "metadata") + .intValue(ANY_COL_2, 2) + .build(); // Act Assert assertThatThrownBy(() -> checker.check(put)) @@ -106,8 +131,14 @@ public void checkForPut_ThatMutatesMetadataColumns_ShouldThrowIllegalArgumentExc public void checkForPut_ThatDoNotMutateMetadataColumns_ShouldDoNothing() throws ExecutionException { // Arrange - Set columns = ImmutableSet.of(ANY_COL_1, ANY_COL_2); - when(put.getContainedColumnNames()).thenReturn(columns); + Put put = + Put.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofInt("pk", 1)) + .intValue(ANY_COL_1, 1) + .intValue(ANY_COL_2, 2) + .build(); // Act Assert assertThatCode(() -> checker.check(put)).doesNotThrowAnyException(); @@ -123,8 +154,13 @@ public void checkForPut_ThatDoNotMutateMetadataColumns_ShouldDoNothing() ConditionBuilder.putIf(ConditionBuilder.column(ANY_COL_1).isNullInt()) .and(ConditionBuilder.column(ANY_METADATA_COL_1).isNullText()) .build(); - when(put.getCondition()).thenReturn(Optional.of(condition)); - when(put.forFullTableName()).thenReturn(Optional.of("ns.tbl")); + Put put = + Put.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofInt("pk", 1)) + .condition(condition) + .build(); // Act Assert assertThatThrownBy(() -> checker.check(put)).isInstanceOf(IllegalArgumentException.class); @@ -139,7 +175,13 @@ public void checkForPut_WithConditionThatDoNotTargetMetadataColumns_ShouldCallCo ConditionBuilder.putIf(ConditionBuilder.column(ANY_COL_1).isNullInt()) .and(ConditionBuilder.column(ANY_COL_2).isNullText()) .build(); - when(put.getCondition()).thenReturn(Optional.of(condition)); + Put put = + Put.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofInt("pk", 1)) + .condition(condition) + .build(); // Act Assert assertThatCode(() -> checker.check(put)).doesNotThrowAnyException(); @@ -152,7 +194,13 @@ public void checkForPut_WithConditionThatDoNotTargetMetadataColumns_ShouldCallCo public void checkForDelete_WithNonAllowedCondition_ShouldThrowIllegalArgumentException( Class putConditionClass) { // Arrange - when(delete.getCondition()).thenReturn(Optional.of(mock(putConditionClass))); + Delete delete = + Delete.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofInt("pk", 1)) + .condition(mock(putConditionClass)) + .build(); // Act Assert assertThatThrownBy(() -> checker.check(delete)).isInstanceOf(IllegalArgumentException.class); @@ -164,7 +212,13 @@ public void checkForDelete_WithAllowedCondition_ShouldCheckCondition( Class deleteConditionClass) { // Arrange MutationCondition condition = mock(deleteConditionClass); - when(delete.getCondition()).thenReturn(Optional.of(condition)); + Delete delete = + Delete.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofInt("pk", 1)) + .condition(condition) + .build(); // Act Assert assertThatCode(() -> checker.check(delete)).doesNotThrowAnyException(); @@ -180,8 +234,13 @@ public void checkForDelete_WithAllowedCondition_ShouldCheckCondition( ConditionBuilder.deleteIf(ConditionBuilder.column(ANY_COL_1).isNullInt()) .and(ConditionBuilder.column(ANY_METADATA_COL_1).isNullText()) .build(); - when(delete.getCondition()).thenReturn(Optional.of(condition)); - when(delete.forFullTableName()).thenReturn(Optional.of("ns.tbl")); + Delete delete = + Delete.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofInt("pk", 1)) + .condition(condition) + .build(); // Act Assert assertThatThrownBy(() -> checker.check(delete)).isInstanceOf(IllegalArgumentException.class); @@ -197,7 +256,13 @@ public void checkForDelete_WithAllowedCondition_ShouldCheckCondition( ConditionBuilder.deleteIf(ConditionBuilder.column(ANY_COL_1).isNullInt()) .and(ConditionBuilder.column(ANY_COL_2).isNullText()) .build(); - when(delete.getCondition()).thenReturn(Optional.of(condition)); + Delete delete = + Delete.newBuilder() + .namespace("ns") + .table("tbl") + .partitionKey(Key.ofInt("pk", 1)) + .condition(condition) + .build(); // Act Assert assertThatCode(() -> checker.check(delete)).doesNotThrowAnyException(); @@ -246,7 +311,10 @@ public void checkForGet_WithMetadataColumnsInCondition_ShouldThrowIllegalArgumen @Test public void checkForGet_IncludeMetadataEnabled_ShouldNotThrowException() { // Arrange - checker = spy(new ConsensusCommitOperationChecker(metadataManager, true)); + checker = + spy( + new ConsensusCommitOperationChecker( + metadataManager, virtualTableInfoManager, storageInfoProvider, true)); Get get = Get.newBuilder() @@ -404,7 +472,10 @@ public void checkForScan_WithMetadataColumnsInOrdering_ShouldThrowIllegalArgumen @Test public void checkForScan_IncludeMetadataEnabled_ShouldNotThrowException() { // Arrange - checker = spy(new ConsensusCommitOperationChecker(metadataManager, true)); + checker = + spy( + new ConsensusCommitOperationChecker( + metadataManager, virtualTableInfoManager, storageInfoProvider, true)); Scan scan = Scan.newBuilder() @@ -631,19 +702,160 @@ public void checkForScan_WithIndexKeyUsingClusteringKeyInSerializable_ShouldNotT } @Test - public void checkForScan_ValidScan_ShouldNotThrowException() { + public void + checkForGet_ForVirtualTableButConsistentReadsNotGuaranteed_ShouldThrowIllegalArgumentException() + throws ExecutionException { + // Arrange + Get get = + Get.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("pk", 1)).build(); + TransactionContext context = + new TransactionContext("txId", null, Isolation.SNAPSHOT, false, false); + + VirtualTableInfo virtualTableInfo = mock(VirtualTableInfo.class); + when(virtualTableInfoManager.getVirtualTableInfo(get)).thenReturn(virtualTableInfo); + + StorageInfo storageInfo = + new StorageInfoImpl("jdbc", MutationAtomicityUnit.STORAGE, Integer.MAX_VALUE, false); + when(storageInfoProvider.getStorageInfo("ns")).thenReturn(storageInfo); + + // Act Assert + assertThatThrownBy(() -> checker.check(get, context)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void checkForGet_ForVirtualTableAndConsistentReadsGuaranteed_ShouldNotThrowException() + throws ExecutionException { + // Arrange + Get get = + Get.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("pk", 1)).build(); + TransactionContext context = + new TransactionContext("txId", null, Isolation.SNAPSHOT, false, false); + + VirtualTableInfo virtualTableInfo = mock(VirtualTableInfo.class); + when(virtualTableInfoManager.getVirtualTableInfo(get)).thenReturn(virtualTableInfo); + + StorageInfo storageInfo = + new StorageInfoImpl("jdbc", MutationAtomicityUnit.STORAGE, Integer.MAX_VALUE, true); + when(storageInfoProvider.getStorageInfo("ns")).thenReturn(storageInfo); + + // Act Assert + assertThatCode(() -> checker.check(get, context)).doesNotThrowAnyException(); + } + + @Test + public void + checkForScan_ForVirtualTableButConsistentReadsNotGuaranteed_ShouldThrowIllegalArgumentException() + throws ExecutionException { // Arrange Scan scan = - Scan.newBuilder() - .namespace("ns") - .table("tbl") - .partitionKey(Key.ofInt("pk", 1)) - .projections(ANY_COL_1, ANY_COL_2) - .build(); + Scan.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("pk", 1)).build(); + TransactionContext context = + new TransactionContext("txId", null, Isolation.SNAPSHOT, false, false); + + VirtualTableInfo virtualTableInfo = mock(VirtualTableInfo.class); + when(virtualTableInfoManager.getVirtualTableInfo(scan)).thenReturn(virtualTableInfo); + + StorageInfo storageInfo = + new StorageInfoImpl("jdbc", MutationAtomicityUnit.STORAGE, Integer.MAX_VALUE, false); + when(storageInfoProvider.getStorageInfo("ns")).thenReturn(storageInfo); + + // Act Assert + assertThatThrownBy(() -> checker.check(scan, context)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void checkForScan_ForVirtualTableAndConsistentReadsGuaranteed_ShouldNotThrowException() + throws ExecutionException { + // Arrange + Scan scan = + Scan.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("pk", 1)).build(); TransactionContext context = new TransactionContext("txId", null, Isolation.SNAPSHOT, false, false); + VirtualTableInfo virtualTableInfo = mock(VirtualTableInfo.class); + when(virtualTableInfoManager.getVirtualTableInfo(scan)).thenReturn(virtualTableInfo); + + StorageInfo storageInfo = + new StorageInfoImpl("jdbc", MutationAtomicityUnit.STORAGE, Integer.MAX_VALUE, true); + when(storageInfoProvider.getStorageInfo("ns")).thenReturn(storageInfo); + // Act Assert assertThatCode(() -> checker.check(scan, context)).doesNotThrowAnyException(); } + + @Test + public void + checkForPut_ForVirtualTableButConsistentReadsNotGuaranteed_ShouldThrowIllegalArgumentException() + throws ExecutionException { + // Arrange + Put put = + Put.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("pk", 1)).build(); + + VirtualTableInfo virtualTableInfo = mock(VirtualTableInfo.class); + when(virtualTableInfoManager.getVirtualTableInfo(put)).thenReturn(virtualTableInfo); + + StorageInfo storageInfo = + new StorageInfoImpl("jdbc", MutationAtomicityUnit.STORAGE, Integer.MAX_VALUE, false); + when(storageInfoProvider.getStorageInfo("ns")).thenReturn(storageInfo); + + // Act Assert + assertThatThrownBy(() -> checker.check(put)).isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void checkForPut_ForVirtualTableAndConsistentReadsGuaranteed_ShouldNotThrowException() + throws ExecutionException { + // Arrange + Put put = + Put.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("pk", 1)).build(); + + VirtualTableInfo virtualTableInfo = mock(VirtualTableInfo.class); + when(virtualTableInfoManager.getVirtualTableInfo(put)).thenReturn(virtualTableInfo); + + StorageInfo storageInfo = + new StorageInfoImpl("jdbc", MutationAtomicityUnit.STORAGE, Integer.MAX_VALUE, true); + when(storageInfoProvider.getStorageInfo("ns")).thenReturn(storageInfo); + + // Act Assert + assertThatCode(() -> checker.check(put)).doesNotThrowAnyException(); + } + + @Test + public void + checkForDelete_ForVirtualTableButConsistentReadsNotGuaranteed_ShouldThrowIllegalArgumentException() + throws ExecutionException { + // Arrange + Delete delete = + Delete.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("pk", 1)).build(); + + VirtualTableInfo virtualTableInfo = mock(VirtualTableInfo.class); + when(virtualTableInfoManager.getVirtualTableInfo(delete)).thenReturn(virtualTableInfo); + + StorageInfo storageInfo = + new StorageInfoImpl("jdbc", MutationAtomicityUnit.STORAGE, Integer.MAX_VALUE, false); + when(storageInfoProvider.getStorageInfo("ns")).thenReturn(storageInfo); + + // Act Assert + assertThatThrownBy(() -> checker.check(delete)).isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void checkForDelete_ForVirtualTableAndConsistentReadsGuaranteed_ShouldNotThrowException() + throws ExecutionException { + // Arrange + Delete delete = + Delete.newBuilder().namespace("ns").table("tbl").partitionKey(Key.ofInt("pk", 1)).build(); + + VirtualTableInfo virtualTableInfo = mock(VirtualTableInfo.class); + when(virtualTableInfoManager.getVirtualTableInfo(delete)).thenReturn(virtualTableInfo); + + StorageInfo storageInfo = + new StorageInfoImpl("jdbc", MutationAtomicityUnit.STORAGE, Integer.MAX_VALUE, true); + when(storageInfoProvider.getStorageInfo("ns")).thenReturn(storageInfo); + + // Act Assert + assertThatCode(() -> checker.check(delete)).doesNotThrowAnyException(); + } } diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtilsTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtilsTest.java index fc63a28e07..88afc6f30e 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtilsTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtilsTest.java @@ -107,6 +107,88 @@ public class ConsensusCommitUtilsTest { .isInstanceOf(IllegalArgumentException.class); } + @Test + public void + buildTransactionMetadataTableMetadata_tableMetadataGiven_shouldCreateTransactionMetadataTableProperly() { + // Arrange + final String ACCOUNT_ID = "account_id"; + final String ACCOUNT_TYPE = "account_type"; + final String BALANCE = "balance"; + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(ACCOUNT_ID, DataType.INT) + .addColumn(ACCOUNT_TYPE, DataType.INT) + .addColumn(BALANCE, DataType.INT) + .addPartitionKey(ACCOUNT_ID) + .addClusteringKey(ACCOUNT_TYPE) + .build(); + + TableMetadata expected = + TableMetadata.newBuilder() + .addColumn(ACCOUNT_ID, DataType.INT) + .addColumn(ACCOUNT_TYPE, DataType.INT) + .addColumn(Attribute.ID, DataType.TEXT) + .addColumn(Attribute.STATE, DataType.INT) + .addColumn(Attribute.VERSION, DataType.INT) + .addColumn(Attribute.PREPARED_AT, DataType.BIGINT) + .addColumn(Attribute.COMMITTED_AT, DataType.BIGINT) + .addColumn(Attribute.BEFORE_PREFIX + Attribute.ID, DataType.TEXT) + .addColumn(Attribute.BEFORE_PREFIX + Attribute.STATE, DataType.INT) + .addColumn(Attribute.BEFORE_PREFIX + Attribute.VERSION, DataType.INT) + .addColumn(Attribute.BEFORE_PREFIX + Attribute.PREPARED_AT, DataType.BIGINT) + .addColumn(Attribute.BEFORE_PREFIX + Attribute.COMMITTED_AT, DataType.BIGINT) + .addColumn(Attribute.BEFORE_PREFIX + BALANCE, DataType.INT) + .addPartitionKey(ACCOUNT_ID) + .addClusteringKey(ACCOUNT_TYPE) + .build(); + + // Act + TableMetadata actual = + ConsensusCommitUtils.buildTransactionMetadataTableMetadata(tableMetadata); + + // Assert + assertThat(actual).isEqualTo(expected); + } + + @Test + public void + buildTransactionMetadataTableMetadata_tableMetadataThatHasTransactionMetaColumnGiven_shouldThrowIllegalArgumentException() { + // Arrange + + // Act Assert + assertThatThrownBy( + () -> + ConsensusCommitUtils.buildTransactionMetadataTableMetadata( + TableMetadata.newBuilder() + .addColumn("col1", DataType.INT) + .addColumn("col2", DataType.INT) + .addColumn(Attribute.ID, DataType.TEXT) // transaction meta column + .addPartitionKey("col1") + .build())) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void + buildTransactionMetadataTableMetadata_tableMetadataThatHasNonPrimaryKeyColumnWithBeforePrefixGiven_shouldThrowIllegalArgumentException() { + // Arrange + + // Act Assert + assertThatThrownBy( + () -> + ConsensusCommitUtils.buildTransactionMetadataTableMetadata( + TableMetadata.newBuilder() + .addColumn("col1", DataType.INT) + .addColumn("col2", DataType.INT) + .addColumn( + Attribute.BEFORE_PREFIX + "col2", + DataType.INT) // non-primary key column with the "before_" prefix + .addPartitionKey("col1") + .build())) + .isInstanceOf(IllegalArgumentException.class); + } + @Test public void getBeforeImageColumnName_tableMetadataGiven_shouldCreateTransactionalTableProperly() { // Arrange diff --git a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminImportTableIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminImportTableIntegrationTestBase.java index e9eb568267..8dd66a3873 100644 --- a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminImportTableIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminImportTableIntegrationTestBase.java @@ -31,7 +31,7 @@ public abstract class DistributedStorageAdminImportTableIntegrationTestBase { private static final Logger logger = LoggerFactory.getLogger(DistributedStorageAdminImportTableIntegrationTestBase.class); - private static final String TEST_NAME = "storage_admin_import_table"; + private static final String TEST_NAME = "storage_import"; private static final String NAMESPACE = "int_test_" + TEST_NAME; protected final List testDataList = new ArrayList<>(); protected DistributedStorageAdmin admin; diff --git a/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionAdminImportTableIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionAdminImportTableIntegrationTestBase.java index 25010b5be3..9277d6f568 100644 --- a/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionAdminImportTableIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/DistributedTransactionAdminImportTableIntegrationTestBase.java @@ -32,24 +32,31 @@ public abstract class DistributedTransactionAdminImportTableIntegrationTestBase private static final Logger logger = LoggerFactory.getLogger(DistributedTransactionAdminImportTableIntegrationTestBase.class); - private static final String TEST_NAME = "tx_admin_import_table"; - private static final String NAMESPACE = "int_test_" + TEST_NAME; + private static final String TEST_NAME = "tx_import"; + private static final String NAMESPACE_BASE_NAME = "int_test_"; private final List testDataList = new ArrayList<>(); protected DistributedTransactionAdmin admin; protected DistributedTransactionManager manager; + protected String namespace; @BeforeAll public void beforeAll() throws Exception { - initialize(TEST_NAME); + String testName = getTestName(); + initialize(testName); + namespace = getNamespaceBaseName() + testName; } protected void initialize(String testName) throws Exception {} + protected String getTestName() { + return TEST_NAME; + } + protected abstract Properties getProperties(String testName); - protected String getNamespace() { - return NAMESPACE; + protected String getNamespaceBaseName() { + return NAMESPACE_BASE_NAME; } protected Map getCreationOptions() { @@ -61,17 +68,21 @@ private void dropTable() throws Exception { if (!testData.isImportableTable()) { dropNonImportableTable(testData.getTableName()); } else { - admin.dropTable(getNamespace(), testData.getTableName()); + admin.dropTable(namespace, getImportedTableName(testData.getTableName())); } } - if (!admin.namespaceExists(getNamespace())) { + if (!admin.namespaceExists(namespace)) { // Create metadata to be able to delete the namespace using the Admin - admin.repairNamespace(getNamespace(), getCreationOptions()); + admin.repairNamespace(namespace, getCreationOptions()); } - admin.dropNamespace(getNamespace()); + admin.dropNamespace(namespace); admin.dropCoordinatorTables(); } + protected String getImportedTableName(String table) { + return table; + } + @BeforeEach protected void setUp() throws Exception { TransactionFactory factory = TransactionFactory.create(getProperties(TEST_NAME)); @@ -129,8 +140,7 @@ public void importTable_ShouldWorkProperly() throws Exception { public void importTable_ForUnsupportedDatabase_ShouldThrowUnsupportedOperationException() throws ExecutionException { // Act Assert - assertThatThrownBy( - () -> admin.importTable(getNamespace(), "unsupported_db", Collections.emptyMap())) + assertThatThrownBy(() -> admin.importTable(namespace, "unsupported_db", getCreationOptions())) .isInstanceOf(UnsupportedOperationException.class); } @@ -138,39 +148,39 @@ private void importTable_ForImportableTable_ShouldImportProperly( String table, Map overrideColumnsType, TableMetadata metadata) throws ExecutionException { // Act - admin.importTable(getNamespace(), table, Collections.emptyMap(), overrideColumnsType); + admin.importTable(namespace, table, getCreationOptions(), overrideColumnsType); // Assert - assertThat(admin.namespaceExists(getNamespace())).isTrue(); - assertThat(admin.tableExists(getNamespace(), table)).isTrue(); - assertThat(admin.getTableMetadata(getNamespace(), table)).isEqualTo(metadata); + assertThat(admin.namespaceExists(namespace)).isTrue(); + assertThat(admin.tableExists(namespace, getImportedTableName(table))).isTrue(); + assertThat(admin.getTableMetadata(namespace, getImportedTableName(table))).isEqualTo(metadata); } private void importTable_ForNonImportableTable_ShouldThrowIllegalArgumentException(String table) { // Act Assert - assertThatThrownBy(() -> admin.importTable(getNamespace(), table, Collections.emptyMap())) + assertThatThrownBy(() -> admin.importTable(namespace, table, getCreationOptions())) .isInstanceOf(IllegalArgumentException.class); } private void importTable_ForNonExistingTable_ShouldThrowIllegalArgumentException() { // Act Assert assertThatThrownBy( - () -> admin.importTable(getNamespace(), "non-existing-table", Collections.emptyMap())) + () -> admin.importTable(namespace, "non-existing-table", getCreationOptions())) .isInstanceOf(IllegalArgumentException.class); } - public void importTable_ForImportedTable_ShouldInsertThenGetCorrectly() + private void importTable_ForImportedTable_ShouldInsertThenGetCorrectly() throws TransactionException { // Arrange List inserts = testDataList.stream() .filter(TestData::isImportableTable) - .map(td -> td.getInsert(getNamespace(), td.getTableName())) + .map(td -> td.getInsert(namespace, getImportedTableName(td.getTableName()))) .collect(Collectors.toList()); List gets = testDataList.stream() .filter(TestData::isImportableTable) - .map(td -> td.getGet(getNamespace(), td.getTableName())) + .map(td -> td.getGet(namespace, getImportedTableName(td.getTableName()))) .collect(Collectors.toList()); // Act diff --git a/integration-test/src/main/java/com/scalar/db/schemaloader/SchemaLoaderImportIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/schemaloader/SchemaLoaderImportIntegrationTestBase.java index 1612bc20de..64d697d7a2 100644 --- a/integration-test/src/main/java/com/scalar/db/schemaloader/SchemaLoaderImportIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/schemaloader/SchemaLoaderImportIntegrationTestBase.java @@ -32,27 +32,28 @@ public abstract class SchemaLoaderImportIntegrationTestBase { private static final Logger logger = LoggerFactory.getLogger(SchemaLoaderImportIntegrationTestBase.class); + private static final String TEST_NAME = "schema_loader_import"; private static final Path CONFIG_FILE_PATH = Paths.get("config.properties").toAbsolutePath(); private static final Path IMPORT_SCHEMA_FILE_PATH = Paths.get("import_schema.json").toAbsolutePath(); - private static final String NAMESPACE_1 = "int_test_" + TEST_NAME + "1"; - private static final String TABLE_1 = "test_table1"; - private static final String NAMESPACE_2 = "int_test_" + TEST_NAME + "2"; - private static final String TABLE_2 = "test_table2"; + private static final String NAMESPACE_BASE_NAME = "int_test_"; + protected static final String TABLE_1 = "test_table1"; + protected static final String TABLE_2 = "test_table2"; private DistributedStorageAdmin storageAdmin; private DistributedTransactionAdmin transactionAdmin; - private String namespace1; - private String namespace2; + protected String namespace1; + protected String namespace2; @BeforeAll public void beforeAll() throws Exception { - initialize(TEST_NAME); - Properties properties = getProperties(TEST_NAME); - namespace1 = getNamespace1(); - namespace2 = getNamespace2(); + String testName = getTestName(); + initialize(testName); + namespace1 = getNamespaceBaseName() + testName + "1"; + namespace2 = getNamespaceBaseName() + testName + "2"; + Properties properties = getProperties(testName); writeConfigFile(properties); writeSchemaFile(IMPORT_SCHEMA_FILE_PATH, getImportSchemaJsonMap()); StorageFactory factory = StorageFactory.create(properties); @@ -68,6 +69,10 @@ public void setUp() throws Exception { protected void initialize(String testName) throws Exception {} + protected String getTestName() { + return TEST_NAME; + } + protected abstract Properties getProperties(String testName); protected void writeConfigFile(Properties properties) throws IOException { @@ -76,12 +81,8 @@ protected void writeConfigFile(Properties properties) throws IOException { } } - protected String getNamespace1() { - return NAMESPACE_1; - } - - protected String getNamespace2() { - return NAMESPACE_2; + protected String getNamespaceBaseName() { + return NAMESPACE_BASE_NAME; } protected Map getImportSchemaJsonMap() { @@ -142,7 +143,7 @@ public void afterAll() throws Exception { } private void dropTablesIfExist() throws Exception { - transactionAdmin.dropTable(namespace1, TABLE_1, true); + transactionAdmin.dropTable(namespace1, getImportedTableName1(), true); transactionAdmin.dropNamespace(namespace1, true); storageAdmin.dropTable(namespace2, TABLE_2, true); storageAdmin.dropNamespace(namespace2, true); @@ -186,9 +187,9 @@ public void importTables_ImportableTablesGiven_ShouldImportProperly() throws Exc // Assert assertThat(exitCode).isEqualTo(0); - assertThat(transactionAdmin.tableExists(namespace1, TABLE_1)).isTrue(); + assertThat(transactionAdmin.tableExists(namespace1, getImportedTableName1())).isTrue(); assertThat(storageAdmin.tableExists(namespace2, TABLE_2)).isTrue(); - assertThat(transactionAdmin.getTableMetadata(namespace1, TABLE_1)) + assertThat(transactionAdmin.getTableMetadata(namespace1, getImportedTableName1())) .isEqualTo(getImportableTableMetadata(true)); assertThat(storageAdmin.getTableMetadata(namespace2, TABLE_2)) .isEqualTo(getImportableTableMetadata(false)); @@ -225,7 +226,7 @@ public void importTables_ImportableTablesAndNonRelatedSameNameTableGiven_ShouldI // Assert assertThat(exitCode).isEqualTo(0); - assertThat(transactionAdmin.tableExists(namespace1, TABLE_1)).isTrue(); + assertThat(transactionAdmin.tableExists(namespace1, getImportedTableName1())).isTrue(); assertThat(storageAdmin.tableExists(namespace2, TABLE_2)).isTrue(); assertThat(transactionAdmin.coordinatorTablesExist()).isFalse(); } finally { @@ -270,6 +271,10 @@ public void importTables_NonImportableTablesGiven_ShouldThrowIllegalArgumentExce dropNonImportableTable(namespace2, TABLE_2); } + protected String getImportedTableName1() { + return TABLE_1; + } + protected int executeWithArgs(List args) { return SchemaLoader.mainInternal(args.toArray(new String[0])); } diff --git a/integration-test/src/main/java/com/scalar/db/schemaloader/SchemaLoaderImportWithMetadataDecouplingIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/schemaloader/SchemaLoaderImportWithMetadataDecouplingIntegrationTestBase.java new file mode 100644 index 0000000000..694de1a500 --- /dev/null +++ b/integration-test/src/main/java/com/scalar/db/schemaloader/SchemaLoaderImportWithMetadataDecouplingIntegrationTestBase.java @@ -0,0 +1,31 @@ +package com.scalar.db.schemaloader; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; + +public abstract class SchemaLoaderImportWithMetadataDecouplingIntegrationTestBase + extends SchemaLoaderImportIntegrationTestBase { + + @Override + protected String getTestName() { + return "schema_loader_import_decoupling"; + } + + @Override + protected Map getImportSchemaJsonMap() { + return ImmutableMap.of( + namespace1 + "." + TABLE_1, + ImmutableMap.builder() + .put("transaction", true) + .put("override-columns-type", getImportableTableOverrideColumnsType()) + .put("transaction-metadata-decoupling", true) + .build(), + namespace2 + "." + TABLE_2, + ImmutableMap.builder().put("transaction", false).build()); + } + + @Override + protected String getImportedTableName1() { + return super.getImportedTableName1() + "_scalardb"; + } +} diff --git a/integration-test/src/main/java/com/scalar/db/schemaloader/SchemaLoaderIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/schemaloader/SchemaLoaderIntegrationTestBase.java index 076c369a47..78f945ad35 100644 --- a/integration-test/src/main/java/com/scalar/db/schemaloader/SchemaLoaderIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/schemaloader/SchemaLoaderIntegrationTestBase.java @@ -41,21 +41,21 @@ public abstract class SchemaLoaderIntegrationTestBase { private static final Logger logger = LoggerFactory.getLogger(SchemaLoaderIntegrationTestBase.class); + private static final String TEST_NAME = "schema_loader"; private static final Path CONFIG_FILE_PATH = Paths.get("config.properties").toAbsolutePath(); private static final Path SCHEMA_FILE_PATH = Paths.get("schema.json").toAbsolutePath(); private static final Path ALTERED_SCHEMA_FILE_PATH = Paths.get("altered_schema.json").toAbsolutePath(); - private static final String NAMESPACE_1 = "int_test_" + TEST_NAME + "1"; - private static final String TABLE_1 = "test_table1"; - private static final String NAMESPACE_2 = "int_test_" + TEST_NAME + "2"; - private static final String TABLE_2 = "test_table2"; + private static final String NAMESPACE_BASE_NAME = "int_test_"; + protected static final String TABLE_1 = "test_table1"; + protected static final String TABLE_2 = "test_table2"; private DistributedStorageAdmin storageAdmin; private DistributedTransactionAdmin transactionAdmin; - private String namespace1; - private String namespace2; + protected String namespace1; + protected String namespace2; private String systemNamespaceName; private AdminTestUtils adminTestUtils; @@ -100,10 +100,11 @@ private TableMetadata getTable2Metadata() { @BeforeAll public void beforeAll() throws Exception { - initialize(TEST_NAME); - Properties properties = getProperties(TEST_NAME); - namespace1 = getNamespace1(); - namespace2 = getNamespace2(); + String testName = getTestName(); + initialize(testName); + namespace1 = getNamespaceBaseName() + testName + "1"; + namespace2 = getNamespaceBaseName() + testName + "2"; + Properties properties = getProperties(testName); writeConfigFile(properties); writeSchemaFile(SCHEMA_FILE_PATH, getSchemaJsonMap()); writeSchemaFile(ALTERED_SCHEMA_FILE_PATH, getAlteredSchemaJsonMap()); @@ -112,7 +113,7 @@ public void beforeAll() throws Exception { TransactionFactory transactionFactory = TransactionFactory.create(properties); transactionAdmin = transactionFactory.getTransactionAdmin(); systemNamespaceName = DatabaseConfig.getSystemNamespaceName(properties); - adminTestUtils = getAdminTestUtils(TEST_NAME); + adminTestUtils = getAdminTestUtils(testName); } @BeforeEach @@ -122,6 +123,10 @@ public void setUp() throws Exception { protected void initialize(String testName) throws Exception {} + protected String getTestName() { + return TEST_NAME; + } + protected abstract Properties getProperties(String testName); protected void writeConfigFile(Properties properties) throws IOException { @@ -130,16 +135,12 @@ protected void writeConfigFile(Properties properties) throws IOException { } } - protected String getNamespace1() { - return NAMESPACE_1; - } - - protected String getNamespace2() { - return NAMESPACE_2; + protected String getNamespaceBaseName() { + return NAMESPACE_BASE_NAME; } private String getCoordinatorNamespaceName() { - return getProperties(TEST_NAME).getProperty(ConsensusCommitConfig.COORDINATOR_NAMESPACE); + return getProperties(getTestName()).getProperty(ConsensusCommitConfig.COORDINATOR_NAMESPACE); } protected Map storageOption() { diff --git a/integration-test/src/main/java/com/scalar/db/schemaloader/SchemaLoaderWithMetadataDecouplingIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/schemaloader/SchemaLoaderWithMetadataDecouplingIntegrationTestBase.java new file mode 100644 index 0000000000..5a56ddc4ef --- /dev/null +++ b/integration-test/src/main/java/com/scalar/db/schemaloader/SchemaLoaderWithMetadataDecouplingIntegrationTestBase.java @@ -0,0 +1,75 @@ +package com.scalar.db.schemaloader; + +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +public abstract class SchemaLoaderWithMetadataDecouplingIntegrationTestBase + extends SchemaLoaderIntegrationTestBase { + + @Override + protected String getTestName() { + return "schema_loader_decoupling"; + } + + @Override + protected Map getSchemaJsonMap() { + return ImmutableMap.of( + namespace1 + "." + TABLE_1, + ImmutableMap.builder() + .put("transaction", true) + .put("partition-key", Collections.singletonList("pk1")) + .put("clustering-key", Arrays.asList("ck1 DESC", "ck2 ASC")) + .put( + "columns", + ImmutableMap.builder() + .put("pk1", "INT") + .put("ck1", "INT") + .put("ck2", "TEXT") + .put("col1", "INT") + .put("col2", "BIGINT") + .put("col3", "FLOAT") + .put("col4", "DOUBLE") + .put("col5", "TEXT") + .put("col6", "BLOB") + .put("col7", "BOOLEAN") + .put("col8", "DATE") + .put("col9", "TIME") + .putAll( + isTimestampTypeSupported() + ? ImmutableMap.of("col10", "TIMESTAMPTZ", "col11", "TIMESTAMP") + : ImmutableMap.of("col10", "TIMESTAMPTZ")) + .build()) + .put("secondary-index", Arrays.asList("col1", "col5")) + .put("transaction-metadata-decoupling", true) + .build(), + namespace2 + "." + TABLE_2, + ImmutableMap.builder() + .put("transaction", false) + .put("partition-key", Collections.singletonList("pk1")) + .put("clustering-key", Collections.singletonList("ck1")) + .put( + "columns", + ImmutableMap.of( + "pk1", "INT", "ck1", "INT", "col1", "INT", "col2", "BIGINT", "col3", "FLOAT")) + .build()); + } + + @Disabled("Repairing Transaction metadata decoupling tables is not supported") + @Test + @Override + public void createTablesThenDropTablesThenRepairAllWithoutCoordinator_ShouldExecuteProperly() {} + + @Disabled("Repairing Transaction metadata decoupling tables is not supported") + @Test + @Override + public void createTablesThenDropTablesThenRepairAllWithCoordinator_ShouldExecuteProperly() {} + + @Disabled("Altering Transaction metadata decoupling tables is not supported") + @Test + @Override + public void createTableThenAlterTables_ShouldExecuteProperly() {} +} diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminImportTableIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminImportTableIntegrationTestBase.java index dcf75be9d8..9e082e484c 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminImportTableIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminImportTableIntegrationTestBase.java @@ -6,6 +6,11 @@ public abstract class ConsensusCommitAdminImportTableIntegrationTestBase extends DistributedTransactionAdminImportTableIntegrationTestBase { + @Override + protected String getTestName() { + return "tx_cc_import"; + } + @Override protected final Properties getProperties(String testName) { Properties properties = new Properties(); diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminImportTableWithMetadataDecouplingIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminImportTableWithMetadataDecouplingIntegrationTestBase.java new file mode 100644 index 0000000000..828273f790 --- /dev/null +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitAdminImportTableWithMetadataDecouplingIntegrationTestBase.java @@ -0,0 +1,25 @@ +package com.scalar.db.transaction.consensuscommit; + +import java.util.HashMap; +import java.util.Map; + +public abstract class ConsensusCommitAdminImportTableWithMetadataDecouplingIntegrationTestBase + extends ConsensusCommitAdminImportTableIntegrationTestBase { + + @Override + protected String getTestName() { + return "tx_cc_import_decoupling"; + } + + @Override + protected Map getCreationOptions() { + Map options = new HashMap<>(super.getCreationOptions()); + options.put("transaction-metadata-decoupling", "true"); + return options; + } + + @Override + protected String getImportedTableName(String table) { + return table + "_scalardb"; + } +} diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitImportTableIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitImportTableIntegrationTestBase.java new file mode 100644 index 0000000000..4d966cbe0c --- /dev/null +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitImportTableIntegrationTestBase.java @@ -0,0 +1,958 @@ +package com.scalar.db.transaction.consensuscommit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +import com.google.common.util.concurrent.Uninterruptibles; +import com.scalar.db.api.Consistency; +import com.scalar.db.api.Delete; +import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.DistributedStorageAdmin; +import com.scalar.db.api.DistributedTransaction; +import com.scalar.db.api.Get; +import com.scalar.db.api.Put; +import com.scalar.db.api.Result; +import com.scalar.db.api.Scan; +import com.scalar.db.api.Selection; +import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.TransactionState; +import com.scalar.db.common.StorageInfoProvider; +import com.scalar.db.config.DatabaseConfig; +import com.scalar.db.exception.storage.ExecutionException; +import com.scalar.db.exception.transaction.TransactionException; +import com.scalar.db.io.BigIntColumn; +import com.scalar.db.io.DataType; +import com.scalar.db.io.IntColumn; +import com.scalar.db.io.Key; +import com.scalar.db.io.TextColumn; +import com.scalar.db.service.StorageFactory; +import com.scalar.db.util.AdminTestUtils; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; +import javax.annotation.Nullable; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class ConsensusCommitImportTableIntegrationTestBase { + + private static final String TEST_NAME = "cc_import"; + private static final String NAMESPACE_BASE_NAME = "int_test_"; + protected static final String TABLE = "test_table"; + private static final String ACCOUNT_ID = "account_id"; + private static final String BALANCE = "balance"; + private static final int INITIAL_BALANCE = 1000; + private static final int NUM_ACCOUNTS = 4; + private static final String ANY_ID_1 = "id1"; + + private static final TableMetadata TABLE_METADATA = + TableMetadata.newBuilder() + .addColumn(ACCOUNT_ID, DataType.INT) + .addColumn(BALANCE, DataType.INT) + .addPartitionKey(ACCOUNT_ID) + .build(); + + private DistributedStorage originalStorage; + private DistributedStorageAdmin admin; + private DatabaseConfig databaseConfig; + private ConsensusCommitConfig consensusCommitConfig; + private ConsensusCommitAdmin consensusCommitAdmin; + protected String namespace; + private ParallelExecutor parallelExecutor; + + private ConsensusCommitManager manager; + private DistributedStorage storage; + private Coordinator coordinator; + private RecoveryHandler recovery; + private RecoveryExecutor recoveryExecutor; + @Nullable private CoordinatorGroupCommitter groupCommitter; + + private AdminTestUtils adminTestUtils; + + @BeforeAll + public void beforeAll() throws Exception { + String testName = getTestName(); + initialize(testName); + + namespace = getNamespaceBaseName() + testName; + + Properties properties = getProperties(testName); + + // Add testName as a coordinator namespace suffix + ConsensusCommitTestUtils.addSuffixToCoordinatorNamespace(properties, testName); + + StorageFactory factory = StorageFactory.create(properties); + admin = factory.getStorageAdmin(); + databaseConfig = new DatabaseConfig(properties); + consensusCommitConfig = new ConsensusCommitConfig(databaseConfig); + consensusCommitAdmin = new ConsensusCommitAdmin(admin, consensusCommitConfig, false); + originalStorage = factory.getStorage(); + parallelExecutor = new ParallelExecutor(consensusCommitConfig); + + adminTestUtils = getAdminTestUtils(testName); + } + + protected void initialize(String testName) throws Exception {} + + protected String getTestName() { + return TEST_NAME; + } + + protected abstract Properties getProperties(String testName); + + protected String getNamespaceBaseName() { + return NAMESPACE_BASE_NAME; + } + + protected Map getCreationOptions() { + return Collections.emptyMap(); + } + + protected abstract AdminTestUtils getAdminTestUtils(String testName); + + @BeforeEach + public void setUp() throws Exception { + dropTables(); + storage = spy(originalStorage); + coordinator = spy(new Coordinator(storage, consensusCommitConfig)); + TransactionTableMetadataManager tableMetadataManager = + new TransactionTableMetadataManager(admin, -1); + recovery = spy(new RecoveryHandler(storage, coordinator, tableMetadataManager)); + recoveryExecutor = new RecoveryExecutor(coordinator, recovery, tableMetadataManager); + groupCommitter = CoordinatorGroupCommitter.from(consensusCommitConfig).orElse(null); + CrudHandler crud = + new CrudHandler( + storage, + recoveryExecutor, + tableMetadataManager, + consensusCommitConfig.isIncludeMetadataEnabled(), + parallelExecutor); + CommitHandler commit = spy(createCommitHandler(tableMetadataManager, groupCommitter)); + manager = + new ConsensusCommitManager( + storage, + admin, + consensusCommitConfig, + databaseConfig, + coordinator, + parallelExecutor, + recoveryExecutor, + crud, + commit, + consensusCommitConfig.getIsolation(), + groupCommitter); + } + + private CommitHandler createCommitHandler( + TransactionTableMetadataManager tableMetadataManager, + @Nullable CoordinatorGroupCommitter groupCommitter) { + MutationsGrouper mutationsGrouper = new MutationsGrouper(new StorageInfoProvider(admin)); + if (groupCommitter != null) { + return new CommitHandlerWithGroupCommit( + storage, + coordinator, + tableMetadataManager, + parallelExecutor, + mutationsGrouper, + true, + false, + groupCommitter); + } else { + return new CommitHandler( + storage, + coordinator, + tableMetadataManager, + parallelExecutor, + mutationsGrouper, + true, + false); + } + } + + @AfterEach + public void tearDown() { + if (groupCommitter != null) { + groupCommitter.close(); + } + } + + @AfterAll + public void afterAll() throws Exception { + dropTables(); + consensusCommitAdmin.close(); + originalStorage.close(); + parallelExecutor.close(); + recoveryExecutor.close(); + adminTestUtils.close(); + } + + private void dropTables() throws ExecutionException { + consensusCommitAdmin.dropTable(namespace, getImportedTableName(), true); + consensusCommitAdmin.dropNamespace(namespace, true); + consensusCommitAdmin.dropCoordinatorTables(true); + } + + protected String getImportedTableName() { + return TABLE; + } + + private void prepareImportedTableAndRecords() throws Exception { + createStorageTable(); + + for (int i = 0; i < NUM_ACCOUNTS; i++) { + Put put = + Put.newBuilder() + .namespace(namespace) + .table(TABLE) + .partitionKey(Key.ofInt(ACCOUNT_ID, i)) + .value(IntColumn.of(BALANCE, INITIAL_BALANCE)) + .build(); + originalStorage.put(put); + } + + adminTestUtils.truncateNamespacesTable(); + adminTestUtils.truncateMetadataTable(); + importTable(); + + consensusCommitAdmin.createCoordinatorTables(true, getCreationOptions()); + + // Wait for cache expiry + Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); + } + + private void prepareImportedTableAndPreparedRecordWithNullAndCoordinatorStateRecord( + TransactionState recordState, long preparedAt, TransactionState coordinatorState) + throws Exception { + createStorageTable(); + adminTestUtils.truncateNamespacesTable(); + adminTestUtils.truncateMetadataTable(); + importTable(); + + Put put = + Put.newBuilder() + .namespace(namespace) + .table(getImportedTableName()) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .value(IntColumn.of(BALANCE, INITIAL_BALANCE)) + .value(TextColumn.of(Attribute.ID, ANY_ID_1)) + .value(IntColumn.of(Attribute.STATE, recordState.get())) + .value(IntColumn.of(Attribute.VERSION, 1)) + .value(BigIntColumn.of(Attribute.PREPARED_AT, preparedAt)) + .value(TextColumn.ofNull(Attribute.BEFORE_ID)) + .value(IntColumn.ofNull(Attribute.BEFORE_STATE)) + .value(IntColumn.of(Attribute.BEFORE_VERSION, 0)) + .value(BigIntColumn.ofNull(Attribute.BEFORE_PREPARED_AT)) + .value(BigIntColumn.ofNull(Attribute.BEFORE_COMMITTED_AT)) + .build(); + originalStorage.put(put); + + consensusCommitAdmin.createCoordinatorTables(true, getCreationOptions()); + + if (coordinatorState == null) { + return; + } + Coordinator.State state = new Coordinator.State(ANY_ID_1, coordinatorState); + coordinator.putState(state); + } + + private void createStorageTable() throws ExecutionException { + Map options = getCreationOptions(); + admin.createNamespace(this.namespace, true, options); + admin.createTable(namespace, TABLE, TABLE_METADATA, true, options); + } + + private void importTable() throws ExecutionException { + Map options = new HashMap<>(getCreationOptions()); + consensusCommitAdmin.importTable(namespace, TABLE, options); + } + + private Get prepareGet(int id) { + return Get.newBuilder() + .namespace(namespace) + .table(getImportedTableName()) + .partitionKey(Key.ofInt(ACCOUNT_ID, id)) + .consistency(Consistency.LINEARIZABLE) + .build(); + } + + private List prepareGets() { + List gets = new ArrayList<>(); + IntStream.range(0, NUM_ACCOUNTS).forEach(i -> gets.add(prepareGet(i))); + return gets; + } + + private Scan prepareScanAll() { + return Scan.newBuilder() + .namespace(namespace) + .table(getImportedTableName()) + .all() + .consistency(Consistency.LINEARIZABLE) + .build(); + } + + private Put preparePut(int id) { + return Put.newBuilder() + .namespace(namespace) + .table(getImportedTableName()) + .partitionKey(Key.ofInt(ACCOUNT_ID, id)) + .consistency(Consistency.LINEARIZABLE) + .build(); + } + + private Put preparePut(int id, int balance) { + return Put.newBuilder() + .namespace(namespace) + .table(getImportedTableName()) + .partitionKey(Key.ofInt(ACCOUNT_ID, id)) + .value(IntColumn.of(BALANCE, balance)) + .consistency(Consistency.LINEARIZABLE) + .build(); + } + + private List preparePuts() { + List puts = new ArrayList<>(); + IntStream.range(0, NUM_ACCOUNTS).forEach(i -> puts.add(preparePut(i))); + return puts; + } + + private Delete prepareDelete(int id) { + return Delete.newBuilder() + .namespace(namespace) + .table(getImportedTableName()) + .partitionKey(Key.ofInt(ACCOUNT_ID, id)) + .consistency(Consistency.LINEARIZABLE) + .build(); + } + + private List prepareDeletes() { + List deletes = new ArrayList<>(); + IntStream.range(0, NUM_ACCOUNTS).forEach(i -> deletes.add(prepareDelete(i))); + return deletes; + } + + private int getBalance(Result result) { + assertThat(result.getColumns()).containsKey(BALANCE); + assertThat(result.getColumns().get(BALANCE).hasNullValue()).isFalse(); + return result.getColumns().get(BALANCE).getIntValue(); + } + + private DistributedTransaction prepareTransfer(int fromId, int toId, int amount) + throws TransactionException { + DistributedTransaction transaction = manager.begin(); + + List gets = prepareGets(); + Optional fromResult = transaction.get(gets.get(fromId)); + assertThat(fromResult).isPresent(); + int fromBalance = getBalance(fromResult.get()) - amount; + Optional toResult = transaction.get(gets.get(toId)); + assertThat(toResult).isPresent(); + int toBalance = getBalance(toResult.get()) + amount; + + List puts = preparePuts(); + Put fromPut = + Put.newBuilder(puts.get(fromId)).value(IntColumn.of(BALANCE, fromBalance)).build(); + Put toPut = Put.newBuilder(puts.get(toId)).value(IntColumn.of(BALANCE, toBalance)).build(); + transaction.put(fromPut); + transaction.put(toPut); + + return transaction; + } + + private DistributedTransaction prepareDeletes(int one, int another) throws TransactionException { + DistributedTransaction transaction = manager.begin(); + + List gets = prepareGets(); + transaction.get(gets.get(one)); + transaction.get(gets.get(another)); + + List deletes = prepareDeletes(); + transaction.delete(deletes.get(one)); + transaction.delete(deletes.get(another)); + + return transaction; + } + + @Test + public void get_GetGivenForCommittedRecord_ShouldReturnRecord() throws Exception { + // Arrange + prepareImportedTableAndRecords(); + DistributedTransaction transaction = manager.begin(); + Get get = prepareGet(0); + + // Act + Optional result = transaction.get(get); + transaction.commit(); + + // Assert + assertThat(result.isPresent()).isTrue(); + Assertions.assertThat( + ((TransactionResult) ((FilteredResult) result.get()).getOriginalResult()).getState()) + .isEqualTo(TransactionState.COMMITTED); + } + + @Test + public void get_CalledTwice_ShouldReturnFromSnapshotInSecondTime() throws Exception { + // Arrange + prepareImportedTableAndRecords(); + DistributedTransaction transaction = manager.begin(); + Get get = prepareGet(0); + + // Act + Optional result1 = transaction.get(get); + Optional result2 = transaction.get(get); + transaction.commit(); + + // Assert + verify(storage).get(any(Get.class)); + assertThat(result1).isEqualTo(result2); + } + + @Test + public void scanAll_ScanAllGivenForCommittedRecord_ShouldReturnRecord() throws Exception { + // Arrange + prepareImportedTableAndRecords(); + DistributedTransaction transaction = manager.begin(); + Scan scanAll = Scan.newBuilder(prepareScanAll()).limit(1).build(); + + // Act + List results = transaction.scan(scanAll); + transaction.commit(); + + // Assert + assertThat(results.size()).isEqualTo(1); + Assertions.assertThat( + ((TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult()).getState()) + .isEqualTo(TransactionState.COMMITTED); + } + + private void selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward( + Selection s) throws Exception { + // Arrange + long current = System.currentTimeMillis(); + prepareImportedTableAndPreparedRecordWithNullAndCoordinatorStateRecord( + TransactionState.PREPARED, current, TransactionState.COMMITTED); + DistributedTransaction transaction = manager.begin(); + + // Act + TransactionResult result; + if (s instanceof Get) { + Optional r = transaction.get((Get) s); + assertThat(r).isPresent(); + result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); + } else { + List results = transaction.scan((Scan) s); + assertThat(results.size()).isEqualTo(1); + result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); + } + + assertThat(result.getId()).isEqualTo(ANY_ID_1); + Assertions.assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); + assertThat(result.getVersion()).isEqualTo(1); + assertThat(result.getCommittedAt()).isGreaterThan(0); + + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollforwardRecord(any(Selection.class), any(TransactionResult.class)); + } + + @Test + public void get_GetGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward() + throws Exception { + Get get = prepareGet(0); + selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward(get); + } + + @Test + public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward() + throws Exception { + Scan scanAll = prepareScanAll(); + selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward(scanAll); + } + + private void selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( + Selection s) throws Exception { + // Arrange + long current = System.currentTimeMillis(); + prepareImportedTableAndPreparedRecordWithNullAndCoordinatorStateRecord( + TransactionState.PREPARED, current, TransactionState.ABORTED); + DistributedTransaction transaction = manager.begin(); + + // Act + TransactionResult result; + if (s instanceof Get) { + Optional r = transaction.get((Get) s); + assertThat(r).isPresent(); + result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); + } else { + List results = transaction.scan((Scan) s); + assertThat(results.size()).isEqualTo(1); + result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); + } + + assertThat(result.getId()).isNull(); + Assertions.assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); + assertThat(result.getVersion()).isEqualTo(0); + assertThat(result.getCommittedAt()).isEqualTo(0); + + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); + } + + @Test + public void get_GetGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback() throws Exception { + Get get = prepareGet(0); + selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback(get); + } + + @Test + public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback() + throws Exception { + Scan scanAll = prepareScanAll(); + selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback(scanAll); + } + + private void + selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + Selection s) throws Exception { + // Arrange + long prepared_at = System.currentTimeMillis(); + prepareImportedTableAndPreparedRecordWithNullAndCoordinatorStateRecord( + TransactionState.PREPARED, prepared_at, null); + DistributedTransaction transaction = manager.begin(); + + // Act + assertThatThrownBy( + () -> { + if (s instanceof Get) { + transaction.get((Get) s); + } else { + transaction.scan((Scan) s); + } + }) + .isInstanceOf(UncommittedRecordException.class); + + transaction.rollback(); + + // Assert + verify(recovery, never()).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery, never()).rollbackRecord(any(Selection.class), any(TransactionResult.class)); + verify(coordinator, never()).putState(any(Coordinator.State.class)); + } + + @Test + public void + get_GetGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction() + throws Exception { + Get get = prepareGet(0); + selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + get); + } + + @Test + public void + scanAll_ScanAllGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction() + throws Exception { + Scan scanAll = prepareScanAll(); + selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + scanAll); + } + + private void + selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + Selection s) throws Exception { + // Arrange + long prepared_at = System.currentTimeMillis() - RecoveryHandler.TRANSACTION_LIFETIME_MILLIS - 1; + prepareImportedTableAndPreparedRecordWithNullAndCoordinatorStateRecord( + TransactionState.PREPARED, prepared_at, null); + DistributedTransaction transaction = manager.begin(); + + // Act + TransactionResult result; + if (s instanceof Get) { + Optional r = transaction.get((Get) s); + assertThat(r).isPresent(); + result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); + } else { + List results = transaction.scan((Scan) s); + assertThat(results.size()).isEqualTo(1); + result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); + } + + assertThat(result.getId()).isNull(); + Assertions.assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); + assertThat(result.getVersion()).isEqualTo(0); + assertThat(result.getCommittedAt()).isEqualTo(0); + + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(coordinator).putState(new Coordinator.State(ANY_ID_1, TransactionState.ABORTED)); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); + } + + @Test + public void get_GetGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() + throws Exception { + Get get = prepareGet(0); + selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + get); + } + + @Test + public void + scanAll_ScanAllGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() + throws Exception { + Scan scanAll = prepareScanAll(); + selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + scanAll); + } + + private void selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( + Selection s) throws Exception { + // Arrange + long current = System.currentTimeMillis(); + prepareImportedTableAndPreparedRecordWithNullAndCoordinatorStateRecord( + TransactionState.DELETED, current, TransactionState.COMMITTED); + DistributedTransaction transaction = manager.begin(); + + // Act + if (s instanceof Get) { + assertThat(transaction.get((Get) s).isPresent()).isFalse(); + } else { + List results = transaction.scan((Scan) s); + assertThat(results.size()).isEqualTo(0); + } + + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollforwardRecord(any(Selection.class), any(TransactionResult.class)); + } + + @Test + public void get_GetGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward() + throws Exception { + Get get = prepareGet(0); + selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward(get); + } + + @Test + public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward() + throws Exception { + Scan scanAll = prepareScanAll(); + selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward(scanAll); + } + + private void selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( + Selection s) throws Exception { + // Arrange + long current = System.currentTimeMillis(); + prepareImportedTableAndPreparedRecordWithNullAndCoordinatorStateRecord( + TransactionState.DELETED, current, TransactionState.ABORTED); + DistributedTransaction transaction = manager.begin(); + + // Act + TransactionResult result; + if (s instanceof Get) { + Optional r = transaction.get((Get) s); + assertThat(r).isPresent(); + result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); + } else { + List results = transaction.scan((Scan) s); + assertThat(results.size()).isEqualTo(1); + result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); + } + + assertThat(result.getId()).isNull(); + Assertions.assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); + assertThat(result.getVersion()).isEqualTo(0); + assertThat(result.getCommittedAt()).isEqualTo(0); + + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); + } + + @Test + public void get_GetGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback() throws Exception { + Get get = prepareGet(0); + selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback(get); + } + + @Test + public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback() + throws Exception { + Scan scanAll = prepareScanAll(); + selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback(scanAll); + } + + private void + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + Selection s) throws Exception { + // Arrange + long prepared_at = System.currentTimeMillis(); + prepareImportedTableAndPreparedRecordWithNullAndCoordinatorStateRecord( + TransactionState.DELETED, prepared_at, null); + DistributedTransaction transaction = manager.begin(); + + // Act + assertThatThrownBy( + () -> { + if (s instanceof Get) { + transaction.get((Get) s); + } else { + transaction.scan((Scan) s); + } + }) + .isInstanceOf(UncommittedRecordException.class); + + transaction.rollback(); + + // Assert + verify(recovery, never()).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery, never()).rollbackRecord(any(Selection.class), any(TransactionResult.class)); + verify(coordinator, never()).putState(any(Coordinator.State.class)); + } + + @Test + public void + get_GetGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction() + throws Exception { + Get get = prepareGet(0); + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + get); + } + + @Test + public void + scanAll_ScanAllGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction() + throws Exception { + Scan scanAll = prepareScanAll(); + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + scanAll); + } + + private void + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + Selection s) throws Exception { + // Arrange + long prepared_at = System.currentTimeMillis() - RecoveryHandler.TRANSACTION_LIFETIME_MILLIS - 1; + prepareImportedTableAndPreparedRecordWithNullAndCoordinatorStateRecord( + TransactionState.DELETED, prepared_at, null); + DistributedTransaction transaction = manager.begin(); + + // Act + TransactionResult result; + if (s instanceof Get) { + Optional r = transaction.get((Get) s); + assertThat(r).isPresent(); + result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); + } else { + List results = transaction.scan((Scan) s); + assertThat(results.size()).isEqualTo(1); + result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); + } + + assertThat(result.getId()).isNull(); + Assertions.assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); + assertThat(result.getVersion()).isEqualTo(0); + assertThat(result.getCommittedAt()).isEqualTo(0); + + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(coordinator).putState(new Coordinator.State(ANY_ID_1, TransactionState.ABORTED)); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); + } + + @Test + public void get_GetGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() + throws Exception { + Get get = prepareGet(0); + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + get); + } + + @Test + public void + scanAll_ScanAllGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() + throws Exception { + Scan scanAll = prepareScanAll(); + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + scanAll); + } + + @Test + public void putAndCommit_PutGivenForExistingAfterRead_ShouldUpdateRecord() throws Exception { + // Arrange + prepareImportedTableAndRecords(); + Get get = prepareGet(0); + DistributedTransaction transaction = manager.begin(); + + // Act + Optional result = transaction.get(get); + assertThat(result).isPresent(); + int expected = getBalance(result.get()) + 100; + Put put = preparePut(0, expected); + transaction.put(put); + transaction.commit(); + + // Assert + DistributedTransaction another = manager.begin(); + Optional r = another.get(get); + another.commit(); + + assertThat(r).isPresent(); + TransactionResult actual = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); + assertThat(getBalance(actual)).isEqualTo(expected); + Assertions.assertThat(actual.getState()).isEqualTo(TransactionState.COMMITTED); + assertThat(actual.getVersion()).isEqualTo(1); + } + + @Test + public void putAndCommit_PutWithImplicitPreReadEnabledGivenForExisting_ShouldUpdateRecord() + throws Exception { + // Arrange + prepareImportedTableAndRecords(); + + DistributedTransaction transaction = manager.begin(); + + // Act + int expected = INITIAL_BALANCE + 100; + Put put = Put.newBuilder(preparePut(0, expected)).enableImplicitPreRead().build(); + transaction.put(put); + transaction.commit(); + + // Assert + DistributedTransaction another = manager.begin(); + Optional r = another.get(prepareGet(0)); + another.commit(); + + assertThat(r).isPresent(); + TransactionResult actual = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); + assertThat(getBalance(actual)).isEqualTo(expected); + Assertions.assertThat(actual.getState()).isEqualTo(TransactionState.COMMITTED); + assertThat(actual.getVersion()).isEqualTo(1); + } + + @Test + public void putAndCommit_GetsAndPutsGiven_ShouldCommitProperly() throws Exception { + // Arrange + prepareImportedTableAndRecords(); + + List gets = prepareGets(); + + int amount = 100; + int fromBalance = INITIAL_BALANCE - amount; + int toBalance = INITIAL_BALANCE + amount; + int from = 0; + int to = 1; + + // Act + prepareTransfer(from, to, amount).commit(); + + // Assert + DistributedTransaction another = manager.begin(); + Optional fromResult = another.get(gets.get(from)); + assertThat(fromResult).isPresent(); + assertThat(fromResult.get().getColumns()).containsKey(BALANCE); + assertThat(fromResult.get().getInt(BALANCE)).isEqualTo(fromBalance); + Optional toResult = another.get(gets.get(to)); + assertThat(toResult).isPresent(); + assertThat(toResult.get().getColumns()).containsKey(BALANCE); + assertThat(toResult.get().getInt(BALANCE)).isEqualTo(toBalance); + another.commit(); + } + + @Test + public void commit_NonConflictingDeletesGivenForExisting_ShouldCommitBoth() throws Exception { + // Arrange + int account1 = 0; + int account2 = 1; + int account3 = 2; + int account4 = 3; + + prepareImportedTableAndRecords(); + + DistributedTransaction transaction = prepareDeletes(account1, account2); + + // Act + assertThatCode(() -> prepareDeletes(account3, account4).commit()).doesNotThrowAnyException(); + + assertThatCode(transaction::commit).doesNotThrowAnyException(); + + // Assert + List gets = prepareGets(); + DistributedTransaction another = manager.begin(); + assertThat(another.get(gets.get(account1)).isPresent()).isFalse(); + assertThat(another.get(gets.get(account2)).isPresent()).isFalse(); + assertThat(another.get(gets.get(account3)).isPresent()).isFalse(); + assertThat(another.get(gets.get(account4)).isPresent()).isFalse(); + another.commit(); + } + + @Test + public void commit_DeleteGivenForExistingAfterRead_ShouldDeleteRecord() throws Exception { + // Arrange + prepareImportedTableAndRecords(); + Get get = prepareGet(0); + Delete delete = prepareDelete(0); + DistributedTransaction transaction = manager.begin(); + + // Act + Optional result = transaction.get(get); + transaction.delete(delete); + transaction.commit(); + + // Assert + assertThat(result.isPresent()).isTrue(); + DistributedTransaction another = manager.begin(); + assertThat(another.get(get).isPresent()).isFalse(); + another.commit(); + } +} diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitImportTableWithMetadataDecouplingIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitImportTableWithMetadataDecouplingIntegrationTestBase.java new file mode 100644 index 0000000000..92068246b9 --- /dev/null +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitImportTableWithMetadataDecouplingIntegrationTestBase.java @@ -0,0 +1,25 @@ +package com.scalar.db.transaction.consensuscommit; + +import java.util.HashMap; +import java.util.Map; + +public abstract class ConsensusCommitImportTableWithMetadataDecouplingIntegrationTestBase + extends ConsensusCommitImportTableIntegrationTestBase { + + @Override + protected String getTestName() { + return "cc_import_decoupling"; + } + + @Override + protected Map getCreationOptions() { + Map options = new HashMap<>(super.getCreationOptions()); + options.put("transaction-metadata-decoupling", "true"); + return options; + } + + @Override + protected String getImportedTableName() { + return TABLE + "_scalardb"; + } +} diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitNullMetadataIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitNullMetadataIntegrationTestBase.java index 62c5d8c1f8..aee3b9ba58 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitNullMetadataIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitNullMetadataIntegrationTestBase.java @@ -19,6 +19,7 @@ import com.scalar.db.api.Scan; import com.scalar.db.api.Selection; import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.TransactionCrudOperable; import com.scalar.db.api.TransactionState; import com.scalar.db.common.StorageInfoProvider; import com.scalar.db.config.DatabaseConfig; @@ -49,9 +50,8 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) public abstract class ConsensusCommitNullMetadataIntegrationTestBase { - private static final String TEST_NAME = "cc_null_metadata"; - private static final String NAMESPACE_1 = "int_test_" + TEST_NAME + "1"; - private static final String NAMESPACE_2 = "int_test_" + TEST_NAME + "2"; + private static final String TEST_NAME = "cc_null"; + private static final String NAMESPACE_BASE_NAME = "int_test_"; private static final String TABLE_1 = "test_table1"; private static final String TABLE_2 = "test_table2"; private static final String ACCOUNT_ID = "account_id"; @@ -67,8 +67,8 @@ public abstract class ConsensusCommitNullMetadataIntegrationTestBase { private DatabaseConfig databaseConfig; private ConsensusCommitConfig consensusCommitConfig; private ConsensusCommitAdmin consensusCommitAdmin; - private String namespace1; - private String namespace2; + protected String namespace1; + protected String namespace2; private ParallelExecutor parallelExecutor; private ConsensusCommitManager manager; @@ -80,34 +80,37 @@ public abstract class ConsensusCommitNullMetadataIntegrationTestBase { @BeforeAll public void beforeAll() throws Exception { - initialize(); - Properties properties = getProperties(TEST_NAME); + String testName = getTestName(); + initialize(testName); + + namespace1 = getNamespaceBaseName() + testName + "1"; + namespace2 = getNamespaceBaseName() + testName + "2"; + + Properties properties = getProperties(testName); // Add testName as a coordinator namespace suffix - ConsensusCommitTestUtils.addSuffixToCoordinatorNamespace(properties, TEST_NAME); + ConsensusCommitTestUtils.addSuffixToCoordinatorNamespace(properties, testName); StorageFactory factory = StorageFactory.create(properties); admin = factory.getStorageAdmin(); databaseConfig = new DatabaseConfig(properties); consensusCommitConfig = new ConsensusCommitConfig(databaseConfig); consensusCommitAdmin = new ConsensusCommitAdmin(admin, consensusCommitConfig, false); - namespace1 = getNamespace1(); - namespace2 = getNamespace2(); createTables(); originalStorage = factory.getStorage(); parallelExecutor = new ParallelExecutor(consensusCommitConfig); } - protected void initialize() throws Exception {} + protected void initialize(String testName) throws Exception {} - protected abstract Properties getProperties(String testName); - - protected String getNamespace1() { - return NAMESPACE_1; + protected String getTestName() { + return TEST_NAME; } - protected String getNamespace2() { - return NAMESPACE_2; + protected abstract Properties getProperties(String testName); + + protected String getNamespaceBaseName() { + return NAMESPACE_BASE_NAME; } private void createTables() throws ExecutionException { @@ -466,34 +469,37 @@ public void scan_ScanGivenForCommittedRecord_ShouldReturnRecord() } @Test - public void get_CalledTwice_ShouldReturnFromSnapshotInSecondTime() + public void getScanner_ScanGivenForCommittedRecord_ShouldReturnRecord() throws TransactionException, ExecutionException { // Arrange populateRecordsWithNullMetadata(namespace1, TABLE_1); DistributedTransaction transaction = manager.begin(); - Get get = prepareGet(0, 0, namespace1, TABLE_1); + Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); // Act - Optional result1 = transaction.get(get); - Optional result2 = transaction.get(get); + List results; + try (TransactionCrudOperable.Scanner scanner = transaction.getScanner(scan)) { + results = scanner.all(); + } transaction.commit(); // Assert - verify(storage).get(any(Get.class)); - assertThat(result1).isEqualTo(result2); + assertThat(results.size()).isEqualTo(1); + Assertions.assertThat( + ((TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult()).getState()) + .isEqualTo(TransactionState.COMMITTED); } @Test - public void - get_CalledTwiceAndAnotherTransactionCommitsInBetween_ShouldReturnFromSnapshotInSecondTime() - throws TransactionException, ExecutionException { + public void get_CalledTwice_ShouldReturnFromSnapshotInSecondTime() + throws TransactionException, ExecutionException { // Arrange + populateRecordsWithNullMetadata(namespace1, TABLE_1); DistributedTransaction transaction = manager.begin(); Get get = prepareGet(0, 0, namespace1, TABLE_1); // Act Optional result1 = transaction.get(get); - populateRecordsWithNullMetadata(namespace1, TABLE_1); Optional result2 = transaction.get(get); transaction.commit(); @@ -556,6 +562,13 @@ public void scan_ScanGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollfor selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward(scan); } + @Test + public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward() + throws ExecutionException, CoordinatorException, TransactionException { + Scan scanAll = prepareScanAll(namespace1, TABLE_1); + selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward(scanAll); + } + private void selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( Selection s) throws ExecutionException, CoordinatorException, TransactionException { // Arrange @@ -605,6 +618,13 @@ public void scan_ScanGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback(scan); } + @Test + public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback() + throws TransactionException, ExecutionException, CoordinatorException { + Scan scanAll = prepareScanAll(namespace1, TABLE_1); + selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback(scanAll); + } + private void selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( Selection s) throws ExecutionException, CoordinatorException, TransactionException { @@ -651,6 +671,15 @@ public void scan_ScanGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( scan); } + @Test + public void + scanAll_ScanAllGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction() + throws ExecutionException, CoordinatorException, TransactionException { + Scan scanAll = prepareScanAll(namespace1, TABLE_1); + selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + scanAll); + } + private void selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( Selection s) throws ExecutionException, CoordinatorException, TransactionException { @@ -705,6 +734,15 @@ public void get_GetGivenForPreparedWhenCoordinatorStateNotExistAndExpired_Should scan); } + @Test + public void + scanAll_ScanAllGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() + throws ExecutionException, CoordinatorException, TransactionException { + Scan scanAll = prepareScanAll(namespace1, TABLE_1); + selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + scanAll); + } + private void selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( Selection s) throws ExecutionException, CoordinatorException, TransactionException { // Arrange @@ -750,6 +788,13 @@ public void scan_ScanGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforw selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward(scan); } + @Test + public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward() + throws ExecutionException, CoordinatorException, TransactionException { + Scan scanAll = prepareScanAll(namespace1, TABLE_1); + selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward(scanAll); + } + private void selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( Selection s) throws ExecutionException, CoordinatorException, TransactionException { // Arrange @@ -799,6 +844,13 @@ public void scan_ScanGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback() selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback(scan); } + @Test + public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback() + throws ExecutionException, CoordinatorException, TransactionException { + Scan scanAll = prepareScanAll(namespace1, TABLE_1); + selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback(scanAll); + } + private void selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( Selection s) throws ExecutionException, CoordinatorException, TransactionException { @@ -845,6 +897,15 @@ public void scan_ScanGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback() scan); } + @Test + public void + scanAll_ScanAllGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction() + throws ExecutionException, CoordinatorException, TransactionException { + Scan scanAll = prepareScanAll(namespace1, TABLE_1); + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + scanAll); + } + private void selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( Selection s) throws ExecutionException, CoordinatorException, TransactionException { @@ -899,6 +960,15 @@ public void get_GetGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldA scan); } + @Test + public void + scanAll_ScanAllGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() + throws ExecutionException, CoordinatorException, TransactionException { + Scan scanAll = prepareScanAll(namespace1, TABLE_1); + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + scanAll); + } + @Test public void getThenScanAndGet_CommitHappenedInBetween_OnlyGetShouldReadRepeatably() throws TransactionException, ExecutionException { @@ -1121,68 +1191,4 @@ public void scanAll_ScanAllGivenForCommittedRecord_ShouldReturnRecord() ((TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult()).getState()) .isEqualTo(TransactionState.COMMITTED); } - - @Test - public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback(scanAll); - } - - @Test - public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward(scanAll); - } - - @Test - public void - scanAll_ScanAllGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - scanAll); - } - - @Test - public void - scanAll_ScanAllGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( - scanAll); - } - - @Test - public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback() - throws TransactionException, ExecutionException, CoordinatorException { - Scan scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback(scanAll); - } - - @Test - public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward(scanAll); - } - - @Test - public void - scanAll_ScanAllGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - scanAll); - } - - @Test - public void - scanAll_ScanAllGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( - scanAll); - } } diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java index 716533854d..01382d7cac 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java @@ -85,8 +85,7 @@ public abstract class ConsensusCommitSpecificIntegrationTestBase { private static final String TEST_NAME = "cc"; - private static final String NAMESPACE_1 = "int_test_" + TEST_NAME + "1"; - private static final String NAMESPACE_2 = "int_test_" + TEST_NAME + "2"; + private static final String NAMESPACE_BASE_NAME = "int_test_"; private static final String TABLE_1 = "test_table1"; private static final String TABLE_2 = "test_table2"; protected static final String ACCOUNT_ID = "account_id"; @@ -105,8 +104,8 @@ public abstract class ConsensusCommitSpecificIntegrationTestBase { private DatabaseConfig databaseConfig; private ConsensusCommitConfig consensusCommitConfig; private ConsensusCommitAdmin consensusCommitAdmin; - private String namespace1; - private String namespace2; + protected String namespace1; + protected String namespace2; private ParallelExecutor parallelExecutor; private DistributedStorage storage; @@ -118,34 +117,37 @@ public abstract class ConsensusCommitSpecificIntegrationTestBase { @BeforeAll void beforeAll() throws Exception { - initialize(); - Properties properties = getProperties(TEST_NAME); + String testName = getTestName(); + initialize(testName); + + namespace1 = getNamespaceBaseName() + testName + "1"; + namespace2 = getNamespaceBaseName() + testName + "2"; + + Properties properties = getProperties(testName); // Add testName as a coordinator namespace suffix - ConsensusCommitTestUtils.addSuffixToCoordinatorNamespace(properties, TEST_NAME); + ConsensusCommitTestUtils.addSuffixToCoordinatorNamespace(properties, testName); StorageFactory factory = StorageFactory.create(properties); admin = factory.getStorageAdmin(); databaseConfig = new DatabaseConfig(properties); consensusCommitConfig = new ConsensusCommitConfig(databaseConfig); consensusCommitAdmin = new ConsensusCommitAdmin(admin, consensusCommitConfig, false); - namespace1 = getNamespace1(); - namespace2 = getNamespace2(); createTables(); originalStorage = factory.getStorage(); parallelExecutor = new ParallelExecutor(consensusCommitConfig); } - protected void initialize() throws Exception {} - - protected abstract Properties getProperties(String testName); + protected void initialize(String testName) throws Exception {} - protected String getNamespace1() { - return NAMESPACE_1; + protected String getTestName() { + return TEST_NAME; } - protected String getNamespace2() { - return NAMESPACE_2; + protected abstract Properties getProperties(String testName); + + protected String getNamespaceBaseName() { + return NAMESPACE_BASE_NAME; } protected TableMetadata getTableMetadata() { diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificWithMetadataDecouplingIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificWithMetadataDecouplingIntegrationTestBase.java new file mode 100644 index 0000000000..37a56981ad --- /dev/null +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificWithMetadataDecouplingIntegrationTestBase.java @@ -0,0 +1,20 @@ +package com.scalar.db.transaction.consensuscommit; + +import java.util.HashMap; +import java.util.Map; + +public abstract class ConsensusCommitSpecificWithMetadataDecouplingIntegrationTestBase + extends ConsensusCommitSpecificIntegrationTestBase { + + @Override + protected String getTestName() { + return "cc_decoupling"; + } + + @Override + protected Map getCreationOptions() { + Map options = new HashMap<>(super.getCreationOptions()); + options.put("transaction-metadata-decoupling", "true"); + return options; + } +} diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitWithMetadataDecouplingIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitWithMetadataDecouplingIntegrationTestBase.java new file mode 100644 index 0000000000..ca2e69b229 --- /dev/null +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitWithMetadataDecouplingIntegrationTestBase.java @@ -0,0 +1,20 @@ +package com.scalar.db.transaction.consensuscommit; + +import java.util.HashMap; +import java.util.Map; + +public abstract class ConsensusCommitWithMetadataDecouplingIntegrationTestBase + extends ConsensusCommitIntegrationTestBase { + + @Override + protected String getTestName() { + return "tx_cc_decoupling"; + } + + @Override + protected Map getCreationOptions() { + Map options = new HashMap<>(super.getCreationOptions()); + options.put("transaction-metadata-decoupling", "true"); + return options; + } +} diff --git a/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaLoader.java b/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaLoader.java index 00b86dff52..3167f11bac 100644 --- a/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaLoader.java +++ b/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaLoader.java @@ -854,7 +854,7 @@ private static void importTables( // Import tables try (SchemaOperator operator = getSchemaOperator(config)) { - operator.importTables(tableSchemaList, options); + operator.importTables(tableSchemaList); } } diff --git a/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaOperator.java b/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaOperator.java index 8ae7a8f31d..885052819b 100644 --- a/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaOperator.java +++ b/schema-loader/src/main/java/com/scalar/db/schemaloader/SchemaOperator.java @@ -439,8 +439,7 @@ private void addNewColumnsToTable( } } - public void importTables(List tableSchemaList, Map options) - throws SchemaLoaderException { + public void importTables(List tableSchemaList) throws SchemaLoaderException { for (ImportTableSchema tableSchema : tableSchemaList) { String namespace = tableSchema.getNamespace(); String table = tableSchema.getTable(); @@ -448,11 +447,13 @@ public void importTables(List tableSchemaList, Map overrideColumnsType = ImmutableMap.of("c1", DataType.INT); when(importTableSchema.getOverrideColumnsType()).thenReturn(overrideColumnsType); + when(importTableSchema.getOptions()).thenReturn(options); // Act - operator.importTables(tableSchemaList, options); + operator.importTables(tableSchemaList); // Assert verify(transactionAdmin, times(3)).importTable("ns", "tb", options, overrideColumnsType); @@ -603,9 +604,10 @@ public void importTables_WithoutTransactionTables_ShouldCallProperMethods() thro when(importTableSchema.getTable()).thenReturn("tb"); Map overrideColumnsType = ImmutableMap.of("c1", DataType.INT); when(importTableSchema.getOverrideColumnsType()).thenReturn(overrideColumnsType); + when(importTableSchema.getOptions()).thenReturn(options); // Act - operator.importTables(tableSchemaList, options); + operator.importTables(tableSchemaList); // Assert verify(storageAdmin, times(3)).importTable("ns", "tb", options, overrideColumnsType);