From 3a6d1fadeae4c256c7fecbb664db14d592a91edb Mon Sep 17 00:00:00 2001 From: brfrn169 Date: Mon, 26 May 2025 20:15:33 +0900 Subject: [PATCH 1/6] Improve read process in Consensus Commit --- .../com/scalar/db/common/error/CoreError.java | 2 + .../CommitMutationComposer.java | 13 +- .../consensuscommit/ConsensusCommit.java | 107 +- .../ConsensusCommitConfig.java | 20 +- .../ConsensusCommitManager.java | 20 +- .../consensuscommit/ConsensusCommitUtils.java | 38 + .../consensuscommit/CrudHandler.java | 137 +- .../PrepareMutationComposer.java | 8 +- .../consensuscommit/RecoveryExecutor.java | 277 ++++ .../consensuscommit/RecoveryHandler.java | 82 +- .../RollbackMutationComposer.java | 75 +- .../transaction/consensuscommit/Snapshot.java | 22 +- .../TransactionTableMetadataManager.java | 3 + .../TwoPhaseConsensusCommit.java | 109 +- .../TwoPhaseConsensusCommitManager.java | 20 +- .../ConsensusCommitConfigTest.java | 16 +- .../ConsensusCommitManagerTest.java | 14 +- .../consensuscommit/ConsensusCommitTest.java | 244 +-- .../ConsensusCommitUtilsTest.java | 86 + .../consensuscommit/CrudHandlerTest.java | 562 +++++-- .../consensuscommit/RecoveryExecutorTest.java | 481 ++++++ .../consensuscommit/RecoveryHandlerTest.java | 176 ++- .../RollbackMutationComposerTest.java | 125 +- .../TwoPhaseConsensusCommitManagerTest.java | 10 +- .../TwoPhaseConsensusCommitTest.java | 244 +-- ...CommitNullMetadataIntegrationTestBase.java | 500 +----- ...nsusCommitSpecificIntegrationTestBase.java | 1382 ++++++++++------- ...nsusCommitSpecificIntegrationTestBase.java | 425 +---- 28 files changed, 2737 insertions(+), 2461 deletions(-) create mode 100644 core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutor.java create mode 100644 core/src/test/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutorTest.java diff --git a/core/src/main/java/com/scalar/db/common/error/CoreError.java b/core/src/main/java/com/scalar/db/common/error/CoreError.java index eb5852e649..987c8ba11c 100644 --- a/core/src/main/java/com/scalar/db/common/error/CoreError.java +++ b/core/src/main/java/com/scalar/db/common/error/CoreError.java @@ -1241,6 +1241,8 @@ public enum CoreError implements ScalarDbError { "Getting the storage information failed. Namespace: %s", "", ""), + CONSENSUS_COMMIT_RECOVERING_RECORDS_FAILED( + Category.INTERNAL_ERROR, "0057", "Recovering records failed. Details: %s", "", ""), // // Errors for the unknown transaction status error category diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/CommitMutationComposer.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/CommitMutationComposer.java index 757503f258..720d976cb1 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/CommitMutationComposer.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/CommitMutationComposer.java @@ -6,6 +6,7 @@ import static com.scalar.db.transaction.consensuscommit.Attribute.STATE; import static com.scalar.db.transaction.consensuscommit.Attribute.toIdValue; import static com.scalar.db.transaction.consensuscommit.Attribute.toStateValue; +import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.getTransactionTableMetadata; import com.google.common.annotations.VisibleForTesting; import com.scalar.db.api.ConditionBuilder; @@ -105,7 +106,7 @@ private Put composePut(Operation base, @Nullable TransactionResult result) // Set before image columns to null if (result != null) { TransactionTableMetadata transactionTableMetadata = - tableMetadataManager.getTransactionTableMetadata(base); + getTransactionTableMetadata(tableMetadataManager, base); LinkedHashSet beforeImageColumnNames = transactionTableMetadata.getBeforeImageColumnNames(); TableMetadata tableMetadata = transactionTableMetadata.getTableMetadata(); @@ -137,8 +138,9 @@ private Key getPartitionKey(Operation base, @Nullable TransactionResult result) assert base instanceof Selection; if (result != null) { // for rollforward in lazy recovery - TransactionTableMetadata metadata = tableMetadataManager.getTransactionTableMetadata(base); - return ScalarDbUtils.getPartitionKey(result, metadata.getTableMetadata()); + TransactionTableMetadata transactionTableMetadata = + getTransactionTableMetadata(tableMetadataManager, base); + return ScalarDbUtils.getPartitionKey(result, transactionTableMetadata.getTableMetadata()); } else { throw new AssertionError( "This path should not be reached since the EXTRA_WRITE strategy is deleted"); @@ -155,8 +157,9 @@ private Optional getClusteringKey(Operation base, @Nullable TransactionResu assert base instanceof Selection; if (result != null) { // for rollforward in lazy recovery - TransactionTableMetadata metadata = tableMetadataManager.getTransactionTableMetadata(base); - return ScalarDbUtils.getClusteringKey(result, metadata.getTableMetadata()); + TransactionTableMetadata transactionTableMetadata = + getTransactionTableMetadata(tableMetadataManager, base); + return ScalarDbUtils.getClusteringKey(result, transactionTableMetadata.getTableMetadata()); } else { throw new AssertionError( "This path should not be reached since the EXTRA_WRITE strategy is deleted"); diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommit.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommit.java index 3a41c11fd9..08571505fb 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommit.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommit.java @@ -24,10 +24,8 @@ import com.scalar.db.exception.transaction.UnsatisfiedConditionException; import com.scalar.db.util.ScalarDbUtils; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.util.Iterator; import java.util.List; import java.util.Optional; -import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.annotation.concurrent.NotThreadSafe; import org.slf4j.Logger; @@ -51,24 +49,19 @@ public class ConsensusCommit extends AbstractDistributedTransaction { private static final Logger logger = LoggerFactory.getLogger(ConsensusCommit.class); private final CrudHandler crud; private final CommitHandler commit; - private final RecoveryHandler recovery; private final ConsensusCommitMutationOperationChecker mutationOperationChecker; @Nullable private final CoordinatorGroupCommitter groupCommitter; - private Runnable beforeRecoveryHook; @SuppressFBWarnings("EI_EXPOSE_REP2") public ConsensusCommit( CrudHandler crud, CommitHandler commit, - RecoveryHandler recovery, ConsensusCommitMutationOperationChecker mutationOperationChecker, @Nullable CoordinatorGroupCommitter groupCommitter) { this.crud = checkNotNull(crud); this.commit = checkNotNull(commit); - this.recovery = checkNotNull(recovery); this.mutationOperationChecker = mutationOperationChecker; this.groupCommitter = groupCommitter; - this.beforeRecoveryHook = () -> {}; } @Override @@ -78,63 +71,18 @@ public String getId() { @Override public Optional get(Get get) throws CrudException { - get = copyAndSetTargetToIfNot(get); - try { - return crud.get(get); - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; - } + return crud.get(copyAndSetTargetToIfNot(get)); } @Override public List scan(Scan scan) throws CrudException { - scan = copyAndSetTargetToIfNot(scan); - try { - return crud.scan(scan); - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; - } + return crud.scan(copyAndSetTargetToIfNot(scan)); } @Override public Scanner getScanner(Scan scan) throws CrudException { scan = copyAndSetTargetToIfNot(scan); - Scanner scanner = crud.getScanner(scan); - - return new Scanner() { - @Override - public Optional one() throws CrudException { - try { - return scanner.one(); - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; - } - } - - @Override - public List all() throws CrudException { - try { - return scanner.all(); - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; - } - } - - @Override - public void close() throws CrudException { - scanner.close(); - } - - @Nonnull - @Override - public Iterator iterator() { - return scanner.iterator(); - } - }; + return crud.getScanner(scan); } /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @@ -143,12 +91,7 @@ public Iterator iterator() { public void put(Put put) throws CrudException { put = copyAndSetTargetToIfNot(put); checkMutation(put); - try { - crud.put(put); - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; - } + crud.put(put); } /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @@ -165,12 +108,7 @@ public void put(List puts) throws CrudException { public void delete(Delete delete) throws CrudException { delete = copyAndSetTargetToIfNot(delete); checkMutation(delete); - try { - crud.delete(delete); - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; - } + crud.delete(delete); } /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @@ -196,12 +134,7 @@ public void upsert(Upsert upsert) throws CrudException { upsert = copyAndSetTargetToIfNot(upsert); Put put = ConsensusCommitUtils.createPutForUpsert(upsert); checkMutation(put); - try { - crud.put(put); - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; - } + crud.put(put); } @Override @@ -222,9 +155,6 @@ public void update(Update update) throws CrudException { // If the condition is not specified, it means that the record does not exist. In this case, // we do nothing - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; } } @@ -257,9 +187,6 @@ public void commit() throws CommitException, UnknownTransactionStatusException { try { crud.readIfImplicitPreReadEnabled(); } catch (CrudConflictException e) { - if (e instanceof UncommittedRecordException) { - lazyRecovery((UncommittedRecordException) e); - } throw new CommitConflictException( CoreError.CONSENSUS_COMMIT_CONFLICT_OCCURRED_WHILE_IMPLICIT_PRE_READ.buildMessage(), e, @@ -269,6 +196,12 @@ public void commit() throws CommitException, UnknownTransactionStatusException { CoreError.CONSENSUS_COMMIT_EXECUTING_IMPLICIT_PRE_READ_FAILED.buildMessage(), e, getId()); } + try { + crud.waitForRecoveryCompletionIfNecessary(); + } catch (CrudException e) { + throw new CommitException(e.getMessage(), e, getId()); + } + commit.commit(crud.getSnapshot(), crud.isReadOnly()); } @@ -295,22 +228,6 @@ CommitHandler getCommitHandler() { return commit; } - @VisibleForTesting - RecoveryHandler getRecoveryHandler() { - return recovery; - } - - @VisibleForTesting - void setBeforeRecoveryHook(Runnable beforeRecoveryHook) { - this.beforeRecoveryHook = beforeRecoveryHook; - } - - private void lazyRecovery(UncommittedRecordException e) { - logger.debug("Recover uncommitted records: {}", e.getResults()); - beforeRecoveryHook.run(); - e.getResults().forEach(r -> recovery.recover(e.getSelection(), r)); - } - private void checkMutation(Mutation mutation) throws CrudException { try { mutationOperationChecker.check(mutation); diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfig.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfig.java index ea61b36c58..cf7e1fea60 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfig.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfig.java @@ -35,12 +35,9 @@ public class ConsensusCommitConfig { public static final String COORDINATOR_WRITE_OMISSION_ON_READ_ONLY_ENABLED = PREFIX + "coordinator.write_omission_on_read_only.enabled"; - + public static final String RECOVERY_EXECUTOR_COUNT = PREFIX + "recovery_executor_count"; public static final String PARALLEL_IMPLICIT_PRE_READ = PREFIX + "parallel_implicit_pre_read.enabled"; - - public static final int DEFAULT_PARALLEL_EXECUTOR_COUNT = 128; - public static final String INCLUDE_METADATA_ENABLED = PREFIX + "include_metadata.enabled"; public static final String COORDINATOR_GROUP_COMMIT_PREFIX = PREFIX + "coordinator.group_commit."; @@ -59,6 +56,9 @@ public class ConsensusCommitConfig { public static final String COORDINATOR_GROUP_COMMIT_METRICS_MONITOR_LOG_ENABLED = COORDINATOR_GROUP_COMMIT_PREFIX + "metrics_monitor_log_enabled"; + public static final int DEFAULT_PARALLEL_EXECUTOR_COUNT = 128; + public static final int DEFAULT_RECOVERY_EXECUTOR_COUNT = 128; + public static final int DEFAULT_COORDINATOR_GROUP_COMMIT_SLOT_CAPACITY = 20; public static final int DEFAULT_COORDINATOR_GROUP_COMMIT_GROUP_SIZE_FIX_TIMEOUT_MILLIS = 40; public static final int DEFAULT_COORDINATOR_GROUP_COMMIT_DELAYED_SLOT_MOVE_TIMEOUT_MILLIS = 1200; @@ -77,9 +77,8 @@ public class ConsensusCommitConfig { private final boolean asyncRollbackEnabled; private final boolean coordinatorWriteOmissionOnReadOnlyEnabled; - + private final int recoveryExecutorCount; private final boolean parallelImplicitPreReadEnabled; - private final boolean isIncludeMetadataEnabled; private final boolean coordinatorGroupCommitEnabled; @@ -149,10 +148,13 @@ public ConsensusCommitConfig(DatabaseConfig databaseConfig) { coordinatorWriteOmissionOnReadOnlyEnabled = getBoolean(properties, COORDINATOR_WRITE_OMISSION_ON_READ_ONLY_ENABLED, true); - parallelImplicitPreReadEnabled = getBoolean(properties, PARALLEL_IMPLICIT_PRE_READ, true); + recoveryExecutorCount = + getInt(properties, RECOVERY_EXECUTOR_COUNT, DEFAULT_RECOVERY_EXECUTOR_COUNT); isIncludeMetadataEnabled = getBoolean(properties, INCLUDE_METADATA_ENABLED, false); + parallelImplicitPreReadEnabled = getBoolean(properties, PARALLEL_IMPLICIT_PRE_READ, true); + coordinatorGroupCommitEnabled = getBoolean(properties, COORDINATOR_GROUP_COMMIT_ENABLED, false); coordinatorGroupCommitSlotCapacity = getInt( @@ -223,6 +225,10 @@ public boolean isCoordinatorWriteOmissionOnReadOnlyEnabled() { return coordinatorWriteOmissionOnReadOnlyEnabled; } + public int getRecoveryExecutorCount() { + return recoveryExecutorCount; + } + public boolean isParallelImplicitPreReadEnabled() { return parallelImplicitPreReadEnabled; } diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java index 4b890d0512..b18ddf8812 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java @@ -52,7 +52,7 @@ public class ConsensusCommitManager extends AbstractDistributedTransactionManage private final TransactionTableMetadataManager tableMetadataManager; private final Coordinator coordinator; private final ParallelExecutor parallelExecutor; - private final RecoveryHandler recovery; + private final RecoveryExecutor recoveryExecutor; protected final CommitHandler commit; private final boolean isIncludeMetadataEnabled; private final ConsensusCommitMutationOperationChecker mutationOperationChecker; @@ -71,7 +71,10 @@ public ConsensusCommitManager( tableMetadataManager = new TransactionTableMetadataManager( admin, databaseConfig.getMetadataCacheExpirationTimeSecs()); - recovery = new RecoveryHandler(storage, coordinator, tableMetadataManager); + RecoveryHandler recovery = new RecoveryHandler(storage, coordinator, tableMetadataManager); + recoveryExecutor = + new RecoveryExecutor( + coordinator, recovery, tableMetadataManager, config.getRecoveryExecutorCount()); groupCommitter = CoordinatorGroupCommitter.from(config).orElse(null); commit = createCommitHandler(); isIncludeMetadataEnabled = config.isIncludeMetadataEnabled(); @@ -90,7 +93,10 @@ protected ConsensusCommitManager(DatabaseConfig databaseConfig) { tableMetadataManager = new TransactionTableMetadataManager( admin, databaseConfig.getMetadataCacheExpirationTimeSecs()); - recovery = new RecoveryHandler(storage, coordinator, tableMetadataManager); + RecoveryHandler recovery = new RecoveryHandler(storage, coordinator, tableMetadataManager); + recoveryExecutor = + new RecoveryExecutor( + coordinator, recovery, tableMetadataManager, config.getRecoveryExecutorCount()); groupCommitter = CoordinatorGroupCommitter.from(config).orElse(null); commit = createCommitHandler(); isIncludeMetadataEnabled = config.isIncludeMetadataEnabled(); @@ -106,7 +112,7 @@ protected ConsensusCommitManager(DatabaseConfig databaseConfig) { DatabaseConfig databaseConfig, Coordinator coordinator, ParallelExecutor parallelExecutor, - RecoveryHandler recovery, + RecoveryExecutor recoveryExecutor, CommitHandler commit, @Nullable CoordinatorGroupCommitter groupCommitter) { super(databaseConfig); @@ -118,7 +124,7 @@ protected ConsensusCommitManager(DatabaseConfig databaseConfig) { admin, databaseConfig.getMetadataCacheExpirationTimeSecs()); this.coordinator = coordinator; this.parallelExecutor = parallelExecutor; - this.recovery = recovery; + this.recoveryExecutor = recoveryExecutor; this.commit = commit; this.groupCommitter = groupCommitter; this.isIncludeMetadataEnabled = config.isIncludeMetadataEnabled(); @@ -246,13 +252,14 @@ DistributedTransaction begin( new CrudHandler( storage, snapshot, + recoveryExecutor, tableMetadataManager, isIncludeMetadataEnabled, parallelExecutor, readOnly, oneOperation); DistributedTransaction transaction = - new ConsensusCommit(crud, commit, recovery, mutationOperationChecker, groupCommitter); + new ConsensusCommit(crud, commit, mutationOperationChecker, groupCommitter); if (readOnly) { transaction = new ReadOnlyDistributedTransaction(transaction); } @@ -511,6 +518,7 @@ public void close() { storage.close(); admin.close(); parallelExecutor.close(); + recoveryExecutor.close(); if (isGroupCommitEnabled()) { assert groupCommitter != null; groupCommitter.close(); diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtils.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtils.java index dd787b9929..d156a38a13 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtils.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtils.java @@ -4,6 +4,7 @@ import com.scalar.db.api.ConditionBuilder; import com.scalar.db.api.Insert; import com.scalar.db.api.MutationCondition; +import com.scalar.db.api.Operation; import com.scalar.db.api.Put; import com.scalar.db.api.PutBuilder; import com.scalar.db.api.TableMetadata; @@ -12,8 +13,11 @@ import com.scalar.db.api.UpdateIfExists; import com.scalar.db.api.Upsert; import com.scalar.db.common.error.CoreError; +import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.exception.transaction.UnsatisfiedConditionException; +import com.scalar.db.io.Column; import com.scalar.db.io.DataType; +import com.scalar.db.io.IntColumn; import java.util.Collections; import java.util.HashSet; import java.util.Map; @@ -314,4 +318,38 @@ public static int getNextTxVersion(@Nullable Integer currentTxVersion) { return currentTxVersion + 1; } } + + static void extractAfterImageColumnsFromBeforeImage( + Map> columns, + TransactionResult result, + Set beforeImageColumnNames) { + result + .getColumns() + .forEach( + (k, v) -> { + if (beforeImageColumnNames.contains(k)) { + String columnName = k.substring(Attribute.BEFORE_PREFIX.length()); + if (columnName.equals(Attribute.VERSION) && v.getIntValue() == 0) { + // Since we use version 0 instead of copying NULL for before_version when updating + // a NULL-transaction-metadata record, we conversely change 0 to NULL for + // rollback. See also PrepareMutationComposer. + columns.put(columnName, IntColumn.ofNull(Attribute.VERSION)); + } else { + columns.put(columnName, v.copyWith(columnName)); + } + } + }); + } + + static TransactionTableMetadata getTransactionTableMetadata( + TransactionTableMetadataManager tableMetadataManager, Operation operation) + throws ExecutionException { + TransactionTableMetadata metadata = tableMetadataManager.getTransactionTableMetadata(operation); + if (metadata == null) { + assert operation.forFullTableName().isPresent(); + throw new IllegalArgumentException( + CoreError.TABLE_NOT_FOUND.buildMessage(operation.forFullTableName().get())); + } + return metadata; + } } diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/CrudHandler.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/CrudHandler.java index 9ce5ec289c..43f1efbc27 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/CrudHandler.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/CrudHandler.java @@ -33,7 +33,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -51,6 +50,7 @@ public class CrudHandler { private static final Logger logger = LoggerFactory.getLogger(CrudHandler.class); private final DistributedStorage storage; private final Snapshot snapshot; + private final RecoveryExecutor recoveryExecutor; private final TransactionTableMetadataManager tableMetadataManager; private final boolean isIncludeMetadataEnabled; private final MutationConditionsValidator mutationConditionsValidator; @@ -65,11 +65,13 @@ public class CrudHandler { private final boolean oneOperation; private final List scanners = new ArrayList<>(); + private final List recoveryResults = new ArrayList<>(); @SuppressFBWarnings("EI_EXPOSE_REP2") public CrudHandler( DistributedStorage storage, Snapshot snapshot, + RecoveryExecutor recoveryExecutor, TransactionTableMetadataManager tableMetadataManager, boolean isIncludeMetadataEnabled, ParallelExecutor parallelExecutor, @@ -77,10 +79,11 @@ public CrudHandler( boolean oneOperation) { this.storage = checkNotNull(storage); this.snapshot = checkNotNull(snapshot); - this.tableMetadataManager = tableMetadataManager; + this.recoveryExecutor = checkNotNull(recoveryExecutor); + this.tableMetadataManager = checkNotNull(tableMetadataManager); this.isIncludeMetadataEnabled = isIncludeMetadataEnabled; this.mutationConditionsValidator = new MutationConditionsValidator(snapshot.getId()); - this.parallelExecutor = parallelExecutor; + this.parallelExecutor = checkNotNull(parallelExecutor); this.readOnly = readOnly; this.oneOperation = oneOperation; } @@ -89,6 +92,7 @@ public CrudHandler( CrudHandler( DistributedStorage storage, Snapshot snapshot, + RecoveryExecutor recoveryExecutor, TransactionTableMetadataManager tableMetadataManager, boolean isIncludeMetadataEnabled, MutationConditionsValidator mutationConditionsValidator, @@ -97,10 +101,11 @@ public CrudHandler( boolean oneOperation) { this.storage = checkNotNull(storage); this.snapshot = checkNotNull(snapshot); - this.tableMetadataManager = tableMetadataManager; + this.recoveryExecutor = checkNotNull(recoveryExecutor); + this.tableMetadataManager = checkNotNull(tableMetadataManager); this.isIncludeMetadataEnabled = isIncludeMetadataEnabled; - this.mutationConditionsValidator = mutationConditionsValidator; - this.parallelExecutor = parallelExecutor; + this.mutationConditionsValidator = checkNotNull(mutationConditionsValidator); + this.parallelExecutor = checkNotNull(parallelExecutor); this.readOnly = readOnly; this.oneOperation = oneOperation; } @@ -146,11 +151,15 @@ void readUnread(@Nullable Snapshot.Key key, Get get) throws CrudException { Optional read(@Nullable Snapshot.Key key, Get get) throws CrudException { Optional result = getFromStorage(get); if (result.isPresent() && !result.get().isCommitted()) { - throw new UncommittedRecordException( - get, - result.get(), - CoreError.CONSENSUS_COMMIT_READ_UNCOMMITTED_RECORD.buildMessage(), - snapshot.getId()); + // Lazy recovery + + if (key == null) { + // Only for a Get with index, the argument `key` is null. In that case, create a key from + // the result + key = new Snapshot.Key(get, result.get()); + } + + result = executeRecovery(key, get, result.get()); } if (!get.getConjunctions().isEmpty()) { @@ -186,6 +195,14 @@ Optional read(@Nullable Snapshot.Key key, Get get) throws Cru return result; } + private Optional executeRecovery( + Snapshot.Key key, Selection selection, TransactionResult result) throws CrudException { + RecoveryExecutor.Result recoveryResult = + recoveryExecutor.execute(key, selection, result, snapshot.getId()); + recoveryResults.add(recoveryResult); + return recoveryResult.recoveredResult; + } + public List scan(Scan originalScan) throws CrudException { List originalProjections = new ArrayList<>(originalScan.getProjections()); Scan scan = (Scan) prepareStorageSelection(originalScan); @@ -211,8 +228,8 @@ private LinkedHashMap scanInternal(Scan scan) Scanner scanner = null; try { if (scan.getLimit() > 0) { - // Since the conjunctions may delete some records from the scan result, it is necessary to - // perform the scan without a limit. + // Since recovery and conjunctions may delete some records from the scan result, it is + // necessary to perform the scan without a limit. scanner = scanFromStorage(Scan.newBuilder(scan).limit(0).build()); } else { scanner = scanFromStorage(scan); @@ -257,15 +274,14 @@ private LinkedHashMap scanInternal(Scan scan) private Optional processScanResult( Snapshot.Key key, Scan scan, TransactionResult result) throws CrudException { + Optional ret; if (!result.isCommitted()) { - throw new UncommittedRecordException( - scan, - result, - CoreError.CONSENSUS_COMMIT_READ_UNCOMMITTED_RECORD.buildMessage(), - snapshot.getId()); + // Lazy recovery + ret = executeRecovery(key, scan, result); + } else { + ret = Optional.of(result); } - Optional ret = Optional.of(result); if (!scan.getConjunctions().isEmpty()) { // Because we also get records whose before images match the conjunctions, we need to check if // the current status of the records actually match the conjunctions. @@ -428,7 +444,7 @@ public void readIfImplicitPreReadEnabled() throws CrudException { } } - private Get createGet(Snapshot.Key key) throws CrudException { + private Get createGet(Snapshot.Key key) { GetBuilder.BuildableGet buildableGet = Get.newBuilder() .namespace(key.getNamespace()) @@ -438,6 +454,49 @@ private Get createGet(Snapshot.Key key) throws CrudException { return (Get) prepareStorageSelection(buildableGet.build()); } + public void waitForRecoveryCompletionIfNecessary() throws CrudException { + for (RecoveryExecutor.Result recoveryResult : recoveryResults) { + try { + if (snapshot.containsKeyInWriteSet(recoveryResult.key) + || snapshot.containsKeyInDeleteSet(recoveryResult.key) + || snapshot.isValidationRequired()) { + recoveryResult.recoveryFuture.get(); + } + } catch (java.util.concurrent.ExecutionException e) { + throw new CrudException( + CoreError.CONSENSUS_COMMIT_RECOVERING_RECORDS_FAILED.buildMessage( + e.getCause().getMessage()), + e.getCause(), + snapshot.getId()); + } catch (Exception e) { + throw new CrudException( + CoreError.CONSENSUS_COMMIT_RECOVERING_RECORDS_FAILED.buildMessage(e.getMessage()), + e, + snapshot.getId()); + } + } + } + + @VisibleForTesting + void waitForRecoveryCompletion() throws CrudException { + for (RecoveryExecutor.Result recoveryResult : recoveryResults) { + try { + recoveryResult.recoveryFuture.get(); + } catch (java.util.concurrent.ExecutionException e) { + throw new CrudException( + CoreError.CONSENSUS_COMMIT_RECOVERING_RECORDS_FAILED.buildMessage( + e.getCause().getMessage()), + e.getCause(), + snapshot.getId()); + } catch (Exception e) { + throw new CrudException( + CoreError.CONSENSUS_COMMIT_RECOVERING_RECORDS_FAILED.buildMessage(e.getMessage()), + e, + snapshot.getId()); + } + } + } + // Although this class is not thread-safe, this method is actually thread-safe because the storage // is thread-safe @VisibleForTesting @@ -594,15 +653,8 @@ private Set convertConjunctions( return converted; } - private Selection prepareStorageSelection(Selection selection) throws CrudException { + private Selection prepareStorageSelection(Selection selection) { selection.clearProjections(); - // Retrieve only the after images columns when including the metadata is disabled, otherwise - // retrieve all the columns - if (!isIncludeMetadataEnabled) { - LinkedHashSet afterImageColumnNames = - getTransactionTableMetadata(selection).getAfterImageColumnNames(); - selection.withProjections(afterImageColumnNames); - } selection.withConsistency(Consistency.LINEARIZABLE); return selection; } @@ -610,16 +662,7 @@ private Selection prepareStorageSelection(Selection selection) throws CrudExcept private TransactionTableMetadata getTransactionTableMetadata(Operation operation) throws CrudException { try { - TransactionTableMetadata metadata = - tableMetadataManager.getTransactionTableMetadata(operation); - if (metadata == null) { - assert operation.forNamespace().isPresent() && operation.forTable().isPresent(); - throw new IllegalArgumentException( - CoreError.TABLE_NOT_FOUND.buildMessage( - ScalarDbUtils.getFullTableName( - operation.forNamespace().get(), operation.forTable().get()))); - } - return metadata; + return ConsensusCommitUtils.getTransactionTableMetadata(tableMetadataManager, operation); } catch (ExecutionException e) { throw new CrudException( CoreError.GETTING_TABLE_METADATA_FAILED.buildMessage(), e, snapshot.getId()); @@ -627,18 +670,8 @@ private TransactionTableMetadata getTransactionTableMetadata(Operation operation } private TableMetadata getTableMetadata(Operation operation) throws CrudException { - try { - TransactionTableMetadata metadata = - tableMetadataManager.getTransactionTableMetadata(operation); - if (metadata == null) { - assert operation.forFullTableName().isPresent(); - throw new IllegalArgumentException( - CoreError.TABLE_NOT_FOUND.buildMessage(operation.forFullTableName().get())); - } - return metadata.getTableMetadata(); - } catch (ExecutionException e) { - throw new CrudException(e.getMessage(), e, snapshot.getId()); - } + TransactionTableMetadata metadata = getTransactionTableMetadata(operation); + return metadata.getTableMetadata(); } @SuppressFBWarnings("EI_EXPOSE_REP") @@ -673,8 +706,8 @@ public ConsensusCommitStorageScanner(Scan scan, List originalProjections this.originalProjections = originalProjections; if (scan.getLimit() > 0) { - // Since the conjunctions may delete some records from the scan result, it is necessary to - // perform the scan without a limit. + // Since recovery and conjunctions may delete some records, it is necessary to perform the + // scan without a limit. scanner = scanFromStorage(Scan.newBuilder(scan).limit(0).build()); } else { scanner = scanFromStorage(scan); diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/PrepareMutationComposer.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/PrepareMutationComposer.java index b0dc2648a4..b6feda0e21 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/PrepareMutationComposer.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/PrepareMutationComposer.java @@ -3,6 +3,7 @@ import static com.scalar.db.transaction.consensuscommit.Attribute.ID; import static com.scalar.db.transaction.consensuscommit.ConsensusCommitOperationAttributes.isInsertModeEnabled; import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.getNextTxVersion; +import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.getTransactionTableMetadata; import com.google.common.annotations.VisibleForTesting; import com.scalar.db.api.ConditionBuilder; @@ -146,8 +147,9 @@ private List> createBeforeColumns(Mutation base, TransactionResult res } private boolean isBeforeRequired(Mutation base, String columnName) throws ExecutionException { - TransactionTableMetadata metadata = tableMetadataManager.getTransactionTableMetadata(base); - return !metadata.getPrimaryKeyColumnNames().contains(columnName) - && metadata.getAfterImageColumnNames().contains(columnName); + TransactionTableMetadata transactionTableMetadata = + getTransactionTableMetadata(tableMetadataManager, base); + return !transactionTableMetadata.getPrimaryKeyColumnNames().contains(columnName) + && transactionTableMetadata.getAfterImageColumnNames().contains(columnName); } } diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutor.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutor.java new file mode 100644 index 0000000000..b860130612 --- /dev/null +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutor.java @@ -0,0 +1,277 @@ +package com.scalar.db.transaction.consensuscommit; + +import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.extractAfterImageColumnsFromBeforeImage; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.common.util.concurrent.Uninterruptibles; +import com.scalar.db.api.Operation; +import com.scalar.db.api.Selection; +import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.TransactionState; +import com.scalar.db.common.ResultImpl; +import com.scalar.db.common.error.CoreError; +import com.scalar.db.exception.storage.ExecutionException; +import com.scalar.db.exception.transaction.CrudException; +import com.scalar.db.io.BigIntColumn; +import com.scalar.db.io.BlobColumn; +import com.scalar.db.io.BooleanColumn; +import com.scalar.db.io.Column; +import com.scalar.db.io.DataType; +import com.scalar.db.io.DateColumn; +import com.scalar.db.io.DoubleColumn; +import com.scalar.db.io.FloatColumn; +import com.scalar.db.io.IntColumn; +import com.scalar.db.io.Key; +import com.scalar.db.io.TextColumn; +import com.scalar.db.io.TimeColumn; +import com.scalar.db.io.TimestampColumn; +import com.scalar.db.io.TimestampTZColumn; +import com.scalar.db.util.ScalarDbUtils; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +public class RecoveryExecutor implements AutoCloseable { + + private final Coordinator coordinator; + private final RecoveryHandler recovery; + private final TransactionTableMetadataManager tableMetadataManager; + private final ExecutorService executorService; + + @SuppressFBWarnings("EI_EXPOSE_REP2") + public RecoveryExecutor( + Coordinator coordinator, + RecoveryHandler recovery, + TransactionTableMetadataManager tableMetadataManager, + int threadPoolSize) { + this.coordinator = Objects.requireNonNull(coordinator); + this.recovery = Objects.requireNonNull(recovery); + this.tableMetadataManager = Objects.requireNonNull(tableMetadataManager); + executorService = + Executors.newFixedThreadPool( + threadPoolSize, + new ThreadFactoryBuilder() + .setNameFormat("recovery-executor-%d") + .setDaemon(true) + .build()); + } + + public Result execute( + Snapshot.Key key, Selection selection, TransactionResult result, String transactionId) + throws CrudException { + assert !result.isCommitted(); + + Optional state = getCoordinatorState(result.getId()); + + Optional recoveredResult = + createRecoveredResult(state, selection, result, transactionId); + + // Recover the record + Future future = + executorService.submit( + () -> { + recovery.recover(selection, result, state); + return null; + }); + + return new Result(key, recoveredResult, future); + } + + private Optional getCoordinatorState(String transactionId) + throws CrudException { + try { + return coordinator.getState(transactionId); + } catch (CoordinatorException e) { + throw new CrudException(e.getMessage(), e, transactionId); + } + } + + private Optional createRecoveredResult( + Optional state, + Selection selection, + TransactionResult result, + String transactionId) + throws CrudException { + throwUncommittedRecordExceptionIfTransactionNotExpired(state, selection, result, transactionId); + + if (!state.isPresent() || state.get().getState() == TransactionState.ABORTED) { + return createRolledBackRecord(selection, result, transactionId); + } else { + assert state.get().getState() == TransactionState.COMMITTED; + return createRolledForwardResult(selection, result, transactionId); + } + } + + private void throwUncommittedRecordExceptionIfTransactionNotExpired( + Optional state, + Selection selection, + TransactionResult result, + String transactionId) + throws UncommittedRecordException { + if (!state.isPresent() && !recovery.isTransactionExpired(result)) { + throw new UncommittedRecordException( + selection, + result, + CoreError.CONSENSUS_COMMIT_READ_UNCOMMITTED_RECORD.buildMessage(), + transactionId); + } + } + + private Optional createRolledBackRecord( + Selection selection, TransactionResult result, String transactionId) throws CrudException { + if (!result.hasBeforeImage()) { + return Optional.empty(); + } + + TransactionTableMetadata transactionTableMetadata = + getTransactionTableMetadata(selection, transactionId); + LinkedHashSet beforeImageColumnNames = + transactionTableMetadata.getBeforeImageColumnNames(); + TableMetadata tableMetadata = transactionTableMetadata.getTableMetadata(); + + Map> columns = new HashMap<>(); + + extractAfterImageColumnsFromBeforeImage(columns, result, beforeImageColumnNames); + + Key partitionKey = ScalarDbUtils.getPartitionKey(result, tableMetadata); + partitionKey.getColumns().forEach(c -> columns.put(c.getName(), c)); + + Optional clusteringKey = ScalarDbUtils.getClusteringKey(result, tableMetadata); + clusteringKey.ifPresent(k -> k.getColumns().forEach(c -> columns.put(c.getName(), c))); + + addNullBeforeImageColumns(columns, beforeImageColumnNames, tableMetadata); + + return Optional.of(new TransactionResult(new ResultImpl(columns, tableMetadata))); + } + + private Optional createRolledForwardResult( + Selection selection, TransactionResult result, String transactionId) throws CrudException { + if (result.getState() == TransactionState.DELETED) { + return Optional.empty(); + } + + assert result.getState() == TransactionState.PREPARED; + + TransactionTableMetadata transactionTableMetadata = + getTransactionTableMetadata(selection, transactionId); + TableMetadata tableMetadata = transactionTableMetadata.getTableMetadata(); + + Map> columns = new HashMap<>(); + result + .getColumns() + .forEach( + (columnName, column) -> { + if (columnName.equals(Attribute.STATE)) { + // Set the state to COMMITTED + columns.put( + Attribute.STATE, + IntColumn.of(Attribute.STATE, TransactionState.COMMITTED.get())); + } else { + columns.put(columnName, column); + } + }); + + long committedAt = getCommittedAt(); + columns.put(Attribute.COMMITTED_AT, BigIntColumn.of(Attribute.COMMITTED_AT, committedAt)); + + addNullBeforeImageColumns( + columns, transactionTableMetadata.getBeforeImageColumnNames(), tableMetadata); + + return Optional.of(new TransactionResult(new ResultImpl(columns, tableMetadata))); + } + + @VisibleForTesting + long getCommittedAt() { + // Use the current time as the committedAt timestamp. Note that this is not the actual + // committedAt timestamp of the record + return System.currentTimeMillis(); + } + + private void addNullBeforeImageColumns( + Map> columns, + LinkedHashSet beforeImageColumnNames, + TableMetadata tableMetadata) { + for (String beforeImageColumnName : beforeImageColumnNames) { + DataType columnDataType = tableMetadata.getColumnDataType(beforeImageColumnName); + switch (columnDataType) { + case BOOLEAN: + columns.put(beforeImageColumnName, BooleanColumn.ofNull(beforeImageColumnName)); + break; + case INT: + columns.put(beforeImageColumnName, IntColumn.ofNull(beforeImageColumnName)); + break; + case BIGINT: + columns.put(beforeImageColumnName, BigIntColumn.ofNull(beforeImageColumnName)); + break; + case FLOAT: + columns.put(beforeImageColumnName, FloatColumn.ofNull(beforeImageColumnName)); + break; + case DOUBLE: + columns.put(beforeImageColumnName, DoubleColumn.ofNull(beforeImageColumnName)); + break; + case TEXT: + columns.put(beforeImageColumnName, TextColumn.ofNull(beforeImageColumnName)); + break; + case BLOB: + columns.put(beforeImageColumnName, BlobColumn.ofNull(beforeImageColumnName)); + break; + case DATE: + columns.put(beforeImageColumnName, DateColumn.ofNull(beforeImageColumnName)); + break; + case TIME: + columns.put(beforeImageColumnName, TimeColumn.ofNull(beforeImageColumnName)); + break; + case TIMESTAMP: + columns.put(beforeImageColumnName, TimestampColumn.ofNull(beforeImageColumnName)); + break; + case TIMESTAMPTZ: + columns.put(beforeImageColumnName, TimestampTZColumn.ofNull(beforeImageColumnName)); + break; + default: + throw new AssertionError("Unknown data type: " + columnDataType); + } + } + } + + private TransactionTableMetadata getTransactionTableMetadata( + Operation operation, String transactionId) throws CrudException { + try { + return ConsensusCommitUtils.getTransactionTableMetadata(tableMetadataManager, operation); + } catch (ExecutionException e) { + throw new CrudException( + CoreError.GETTING_TABLE_METADATA_FAILED.buildMessage(), e, transactionId); + } + } + + @Override + public void close() { + executorService.shutdown(); + Uninterruptibles.awaitTerminationUninterruptibly(executorService); + } + + public static class Result { + public final Snapshot.Key key; + + // The recovered result + public final Optional recoveredResult; + + // The future that completes when the recovery is done + public final Future recoveryFuture; + + public Result( + Snapshot.Key key, + Optional recoveredResult, + Future recoveryFuture) { + this.key = key; + this.recoveredResult = recoveredResult; + this.recoveryFuture = recoveryFuture; + } + } +} diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryHandler.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryHandler.java index ef7fc8d0f3..0709ba1bd5 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryHandler.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryHandler.java @@ -8,6 +8,7 @@ import com.scalar.db.api.Selection; import com.scalar.db.api.TransactionState; import com.scalar.db.exception.storage.ExecutionException; +import com.scalar.db.exception.storage.NoMutationException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.List; import java.util.Optional; @@ -17,7 +18,7 @@ @ThreadSafe public class RecoveryHandler { - static final long TRANSACTION_LIFETIME_MILLIS = 15000; + @VisibleForTesting static final long TRANSACTION_LIFETIME_MILLIS = 15000; private static final Logger logger = LoggerFactory.getLogger(RecoveryHandler.class); private final DistributedStorage storage; private final Coordinator coordinator; @@ -33,18 +34,11 @@ public RecoveryHandler( this.tableMetadataManager = checkNotNull(tableMetadataManager); } - // lazy recovery in read phase - public void recover(Selection selection, TransactionResult result) { + public void recover( + Selection selection, TransactionResult result, Optional state) + throws ExecutionException, CoordinatorException { logger.debug("Recovering for {}", result.getId()); - Optional state; - try { - state = coordinator.getState(result.getId()); - } catch (CoordinatorException e) { - logger.warn("Can't get coordinator state. Transaction ID: {}", result.getId(), e); - return; - } - if (state.isPresent()) { if (state.get().getState().equals(TransactionState.COMMITTED)) { rollforwardRecord(selection, result); @@ -57,53 +51,74 @@ public void recover(Selection selection, TransactionResult result) { } @VisibleForTesting - void rollbackRecord(Selection selection, TransactionResult result) { + void rollbackRecord(Selection selection, TransactionResult result) throws ExecutionException { logger.debug( "Rollback for {}, {} mutated by {}", selection.getPartitionKey(), selection.getClusteringKey(), result.getId()); + + RollbackMutationComposer composer = createRollbackMutationComposer(selection, result); + try { - RollbackMutationComposer composer = - new RollbackMutationComposer(result.getId(), storage, tableMetadataManager); - composer.add(selection, result); mutate(composer.get()); - } catch (Exception e) { - logger.warn("Rolling back a record failed. Transaction ID: {}", result.getId(), e); - // ignore since the record is recovered lazily + } catch (NoMutationException ignored) { + // This can happen when the record has already been rolled back by another transaction. In + // this case, we just ignore it. } } @VisibleForTesting - void rollforwardRecord(Selection selection, TransactionResult result) { + RollbackMutationComposer createRollbackMutationComposer( + Selection selection, TransactionResult result) throws ExecutionException { + RollbackMutationComposer composer = + new RollbackMutationComposer(result.getId(), storage, tableMetadataManager); + composer.add(selection, result); + return composer; + } + + @VisibleForTesting + void rollforwardRecord(Selection selection, TransactionResult result) throws ExecutionException { logger.debug( "Rollforward for {}, {} mutated by {}", selection.getPartitionKey(), selection.getClusteringKey(), result.getId()); + + CommitMutationComposer composer = createCommitMutationComposer(selection, result); + try { - CommitMutationComposer composer = - new CommitMutationComposer(result.getId(), tableMetadataManager); - composer.add(selection, result); mutate(composer.get()); - } catch (Exception e) { - logger.warn("Rolling forward a record failed. Transaction ID: {}", result.getId(), e); - // ignore since the record is recovered lazily + } catch (NoMutationException ignored) { + // This can happen when the record has already been committed by another transaction. In this + // case, we just ignore it. } } - private void abortIfExpired(Selection selection, TransactionResult result) { - long current = System.currentTimeMillis(); - if (current <= result.getPreparedAt() + TRANSACTION_LIFETIME_MILLIS) { + @VisibleForTesting + CommitMutationComposer createCommitMutationComposer(Selection selection, TransactionResult result) + throws ExecutionException { + CommitMutationComposer composer = + new CommitMutationComposer(result.getId(), tableMetadataManager); + composer.add(selection, result); + return composer; + } + + private void abortIfExpired(Selection selection, TransactionResult result) + throws CoordinatorException, ExecutionException { + if (!isTransactionExpired(result)) { return; } try { coordinator.putStateForLazyRecoveryRollback(result.getId()); - rollbackRecord(selection, result); - } catch (CoordinatorException e) { - logger.warn("Coordinator tries to abort {}, but failed", result.getId(), e); + } catch (CoordinatorConflictException ignored) { + // This can happen when the record has already been rolled back by another transaction. In + // this case, we just ignore it. + return; } + + rollbackRecord(selection, result); } private void mutate(List mutations) throws ExecutionException { @@ -112,4 +127,9 @@ private void mutate(List mutations) throws ExecutionException { } storage.mutate(mutations); } + + public boolean isTransactionExpired(TransactionResult result) { + long current = System.currentTimeMillis(); + return current > result.getPreparedAt() + TRANSACTION_LIFETIME_MILLIS; + } } diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposer.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposer.java index 13d6d94652..25d9166b78 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposer.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposer.java @@ -5,6 +5,7 @@ import static com.scalar.db.transaction.consensuscommit.Attribute.STATE; import static com.scalar.db.transaction.consensuscommit.Attribute.toIdValue; import static com.scalar.db.transaction.consensuscommit.Attribute.toStateValue; +import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.*; import com.scalar.db.api.ConditionBuilder; import com.scalar.db.api.ConditionalExpression; @@ -22,13 +23,12 @@ import com.scalar.db.api.TransactionState; import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.io.Column; -import com.scalar.db.io.IntColumn; import com.scalar.db.io.Key; import com.scalar.db.util.ScalarDbUtils; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedHashSet; -import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import javax.annotation.Nullable; @@ -46,23 +46,28 @@ public RollbackMutationComposer( this.storage = storage; } - /** rollback in either prepare phase in commit or lazy recovery phase in read */ + /** Rollback in either prepare phase in commit or lazy recovery phase in read. */ @Override public void add(Operation base, @Nullable TransactionResult result) throws ExecutionException { - // We always re-read the latest record here because of the following reasons: - // 1. for usual rollback, we need to check the latest status of the record - // 2. for rollback in lazy recovery, the result doesn't have before image columns - TransactionResult latest = getLatestResult(base, result).orElse(null); - if (latest == null) { - // the record was not prepared (yet) by this transaction or has already been rollback deleted - return; - } - if (!Objects.equals(latest.getId(), id)) { - // This is the case for the record that was not prepared (yet) by this transaction or has - // already been rolled back. We need to use Objects.equals() here since the transaction ID of - // the latest record can be NULL (and different from this transaction's ID) when the record - // has already been rolled back to the deemed committed state by another transaction. - return; + TransactionResult latest; + if (result == null || !Objects.equals(result.getId(), id)) { + // For rollback in prepare phase, we need to check the latest status of the record. + latest = getLatestResult(base, result).orElse(null); + if (latest == null) { + // The record was not prepared (yet) by this transaction or has already been rollback + // deleted. + return; + } + if (!Objects.equals(latest.getId(), id)) { + // This is the case for the record that was not prepared (yet) by this transaction or has + // already been rolled back. We need to use Objects.equals() here since the transaction ID + // of the latest record can be NULL (and different from this transaction's ID) when the + // record has already been rolled back to the deemed committed state by another transaction. + return; + } + } else { + // For rollback in lazy recovery, we can use the result directly. + latest = result; } if (latest.hasBeforeImage()) { @@ -79,29 +84,11 @@ private Put composePut(Operation base, TransactionResult result) throws Executio || result.getState().equals(TransactionState.DELETED)); TransactionTableMetadata transactionTableMetadata = - tableMetadataManager.getTransactionTableMetadata(base); + getTransactionTableMetadata(tableMetadataManager, base); LinkedHashSet beforeImageColumnNames = transactionTableMetadata.getBeforeImageColumnNames(); TableMetadata tableMetadata = transactionTableMetadata.getTableMetadata(); - List> columns = new ArrayList<>(); - result - .getColumns() - .forEach( - (k, v) -> { - if (beforeImageColumnNames.contains(k)) { - String key = k.substring(Attribute.BEFORE_PREFIX.length()); - if (key.equals(Attribute.VERSION) && v.getIntValue() == 0) { - // Since we use version 0 instead of copying NULL for before_version when updating - // a NULL-transaction-metadata record, we conversely change 0 to NULL for - // rollback. See also PrepareMutationComposer. - columns.add(IntColumn.ofNull(Attribute.VERSION)); - } else { - columns.add(v.copyWith(key)); - } - } - }); - Key partitionKey = ScalarDbUtils.getPartitionKey(result, tableMetadata); Optional clusteringKey = ScalarDbUtils.getClusteringKey(result, tableMetadata); @@ -116,7 +103,10 @@ private Put composePut(Operation base, TransactionResult result) throws Executio .build()) .consistency(Consistency.LINEARIZABLE); clusteringKey.ifPresent(putBuilder::clusteringKey); - columns.forEach(putBuilder::value); + + Map> columns = new HashMap<>(); + extractAfterImageColumnsFromBeforeImage(columns, result, beforeImageColumnNames); + columns.values().forEach(putBuilder::value); // Set before image columns to null setBeforeImageColumnsToNull(putBuilder, beforeImageColumnNames, tableMetadata); @@ -129,10 +119,11 @@ private Delete composeDelete(Operation base, TransactionResult result) throws Ex && (result.getState().equals(TransactionState.PREPARED) || result.getState().equals(TransactionState.DELETED)); - TransactionTableMetadata metadata = tableMetadataManager.getTransactionTableMetadata(base); - Key partitionKey = ScalarDbUtils.getPartitionKey(result, metadata.getTableMetadata()); - Optional clusteringKey = - ScalarDbUtils.getClusteringKey(result, metadata.getTableMetadata()); + TransactionTableMetadata transactionTableMetadata = + getTransactionTableMetadata(tableMetadataManager, base); + TableMetadata tableMetadata = transactionTableMetadata.getTableMetadata(); + Key partitionKey = ScalarDbUtils.getPartitionKey(result, tableMetadata); + Optional clusteringKey = ScalarDbUtils.getClusteringKey(result, tableMetadata); return new Delete(partitionKey, clusteringKey.orElse(null)) .forNamespace(base.forNamespace().get()) diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/Snapshot.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/Snapshot.java index 64c3e43b5c..cfd6dc4982 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/Snapshot.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/Snapshot.java @@ -2,6 +2,7 @@ import static com.scalar.db.transaction.consensuscommit.ConsensusCommitOperationAttributes.isImplicitPreReadEnabled; import static com.scalar.db.transaction.consensuscommit.ConsensusCommitOperationAttributes.isInsertModeEnabled; +import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.getTransactionTableMetadata; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; @@ -217,6 +218,14 @@ public boolean containsKeyInGetSet(Get get) { return getSet.containsKey(get); } + public boolean containsKeyInWriteSet(Key key) { + return writeSet.containsKey(key); + } + + public boolean containsKeyInDeleteSet(Key key) { + return deleteSet.containsKey(key); + } + public boolean hasWritesOrDeletes() { return !writeSet.isEmpty() || !deleteSet.isEmpty(); } @@ -235,8 +244,7 @@ public Optional getResult(Key key, Get get) throws CrudExcept return mergeResult(key, result, get.getConjunctions()); } - public Optional> getResults(Scan scan) - throws CrudException { + public Optional> getResults(Scan scan) { if (!scanSet.containsKey(scan)) { return Optional.empty(); } @@ -723,13 +731,9 @@ private void validateGetResult( } private TableMetadata getTableMetadata(Operation operation) throws ExecutionException { - TransactionTableMetadata metadata = tableMetadataManager.getTransactionTableMetadata(operation); - if (metadata == null) { - assert operation.forFullTableName().isPresent(); - throw new IllegalArgumentException( - CoreError.TABLE_NOT_FOUND.buildMessage(operation.forFullTableName().get())); - } - return metadata.getTableMetadata(); + TransactionTableMetadata transactionTableMetadata = + getTransactionTableMetadata(tableMetadataManager, operation); + return transactionTableMetadata.getTableMetadata(); } private boolean isChanged( diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TransactionTableMetadataManager.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TransactionTableMetadataManager.java index e227736134..b77e3a1ee1 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TransactionTableMetadataManager.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TransactionTableMetadataManager.java @@ -13,6 +13,7 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; @ThreadSafe @@ -50,6 +51,7 @@ public Optional load(@Nonnull TableKey key) * @return a table metadata. null if the table is not found. * @throws ExecutionException if the operation fails */ + @Nullable public TransactionTableMetadata getTransactionTableMetadata(Operation operation) throws ExecutionException { if (!operation.forNamespace().isPresent() || !operation.forTable().isPresent()) { @@ -67,6 +69,7 @@ public TransactionTableMetadata getTransactionTableMetadata(Operation operation) * @return a table metadata. null if the table is not found. * @throws ExecutionException if the operation fails */ + @Nullable public TransactionTableMetadata getTransactionTableMetadata(String namespace, String table) throws ExecutionException { try { diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommit.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommit.java index 5c0126d59a..36b32e1791 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommit.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommit.java @@ -27,10 +27,8 @@ import com.scalar.db.exception.transaction.ValidationException; import com.scalar.db.util.ScalarDbUtils; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.util.Iterator; import java.util.List; import java.util.Optional; -import javax.annotation.Nonnull; import javax.annotation.concurrent.NotThreadSafe; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,23 +39,17 @@ public class TwoPhaseConsensusCommit extends AbstractTwoPhaseCommitTransaction { private final CrudHandler crud; private final CommitHandler commit; - private final RecoveryHandler recovery; private final ConsensusCommitMutationOperationChecker mutationOperationChecker; private boolean validated; private boolean needRollback; - // For test - private Runnable beforeRecoveryHook = () -> {}; - @SuppressFBWarnings("EI_EXPOSE_REP2") public TwoPhaseConsensusCommit( CrudHandler crud, CommitHandler commit, - RecoveryHandler recovery, ConsensusCommitMutationOperationChecker mutationOperationChecker) { this.crud = crud; this.commit = commit; - this.recovery = recovery; this.mutationOperationChecker = mutationOperationChecker; } @@ -68,63 +60,17 @@ public String getId() { @Override public Optional get(Get get) throws CrudException { - get = copyAndSetTargetToIfNot(get); - try { - return crud.get(get); - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; - } + return crud.get(copyAndSetTargetToIfNot(get)); } @Override public List scan(Scan scan) throws CrudException { - scan = copyAndSetTargetToIfNot(scan); - try { - return crud.scan(scan); - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; - } + return crud.scan(copyAndSetTargetToIfNot(scan)); } @Override public Scanner getScanner(Scan scan) throws CrudException { - scan = copyAndSetTargetToIfNot(scan); - Scanner scanner = crud.getScanner(scan); - - return new Scanner() { - @Override - public Optional one() throws CrudException { - try { - return scanner.one(); - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; - } - } - - @Override - public List all() throws CrudException { - try { - return scanner.all(); - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; - } - } - - @Override - public void close() throws CrudException { - scanner.close(); - } - - @Nonnull - @Override - public Iterator iterator() { - return scanner.iterator(); - } - }; + return crud.getScanner(copyAndSetTargetToIfNot(scan)); } /** @deprecated As of release 3.13.0. Will be removed in release 5.0.0. */ @@ -147,12 +93,7 @@ public void put(List puts) throws CrudException { private void putInternal(Put put) throws CrudException { put = copyAndSetTargetToIfNot(put); checkMutation(put); - try { - crud.put(put); - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; - } + crud.put(put); } @Override @@ -173,12 +114,7 @@ public void delete(List deletes) throws CrudException { private void deleteInternal(Delete delete) throws CrudException { delete = copyAndSetTargetToIfNot(delete); checkMutation(delete); - try { - crud.delete(delete); - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; - } + crud.delete(delete); } @Override @@ -194,12 +130,7 @@ public void upsert(Upsert upsert) throws CrudException { upsert = copyAndSetTargetToIfNot(upsert); Put put = ConsensusCommitUtils.createPutForUpsert(upsert); checkMutation(put); - try { - crud.put(put); - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; - } + crud.put(put); } @Override @@ -220,9 +151,6 @@ public void update(Update update) throws CrudException { // If the condition is not specified, it means that the record does not exist. In this case, // we do nothing - } catch (UncommittedRecordException e) { - lazyRecovery(e); - throw e; } } @@ -256,9 +184,6 @@ public void prepare() throws PreparationException { try { crud.readIfImplicitPreReadEnabled(); } catch (CrudConflictException e) { - if (e instanceof UncommittedRecordException) { - lazyRecovery((UncommittedRecordException) e); - } throw new PreparationConflictException( CoreError.CONSENSUS_COMMIT_CONFLICT_OCCURRED_WHILE_IMPLICIT_PRE_READ.buildMessage(), e, @@ -268,6 +193,12 @@ public void prepare() throws PreparationException { CoreError.CONSENSUS_COMMIT_EXECUTING_IMPLICIT_PRE_READ_FAILED.buildMessage(), e, getId()); } + try { + crud.waitForRecoveryCompletionIfNecessary(); + } catch (CrudException e) { + throw new PreparationException(e.getMessage(), e, getId()); + } + try { commit.prepareRecords(crud.getSnapshot()); } finally { @@ -338,22 +269,6 @@ CommitHandler getCommitHandler() { return commit; } - @VisibleForTesting - RecoveryHandler getRecoveryHandler() { - return recovery; - } - - @VisibleForTesting - void setBeforeRecoveryHook(Runnable beforeRecoveryHook) { - this.beforeRecoveryHook = beforeRecoveryHook; - } - - private void lazyRecovery(UncommittedRecordException e) { - logger.debug("Recover uncommitted records: {}", e.getResults()); - beforeRecoveryHook.run(); - e.getResults().forEach(r -> recovery.recover(e.getSelection(), r)); - } - private void checkMutation(Mutation mutation) throws CrudException { try { mutationOperationChecker.check(mutation); diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java index eb06445e67..f8e4e30793 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java @@ -55,7 +55,7 @@ public class TwoPhaseConsensusCommitManager extends AbstractTwoPhaseCommitTransa private final TransactionTableMetadataManager tableMetadataManager; private final Coordinator coordinator; private final ParallelExecutor parallelExecutor; - private final RecoveryHandler recovery; + private final RecoveryExecutor recoveryExecutor; private final CommitHandler commit; private final boolean isIncludeMetadataEnabled; private final ConsensusCommitMutationOperationChecker mutationOperationChecker; @@ -73,7 +73,10 @@ public TwoPhaseConsensusCommitManager( admin, databaseConfig.getMetadataCacheExpirationTimeSecs()); coordinator = new Coordinator(storage, config); parallelExecutor = new ParallelExecutor(config); - recovery = new RecoveryHandler(storage, coordinator, tableMetadataManager); + RecoveryHandler recovery = new RecoveryHandler(storage, coordinator, tableMetadataManager); + recoveryExecutor = + new RecoveryExecutor( + coordinator, recovery, tableMetadataManager, config.getRecoveryExecutorCount()); commit = new CommitHandler( storage, @@ -96,7 +99,10 @@ public TwoPhaseConsensusCommitManager(DatabaseConfig databaseConfig) { admin, databaseConfig.getMetadataCacheExpirationTimeSecs()); coordinator = new Coordinator(storage, config); parallelExecutor = new ParallelExecutor(config); - recovery = new RecoveryHandler(storage, coordinator, tableMetadataManager); + RecoveryHandler recovery = new RecoveryHandler(storage, coordinator, tableMetadataManager); + recoveryExecutor = + new RecoveryExecutor( + coordinator, recovery, tableMetadataManager, config.getRecoveryExecutorCount()); commit = new CommitHandler( storage, @@ -117,7 +123,7 @@ public TwoPhaseConsensusCommitManager(DatabaseConfig databaseConfig) { DatabaseConfig databaseConfig, Coordinator coordinator, ParallelExecutor parallelExecutor, - RecoveryHandler recovery, + RecoveryExecutor recoveryExecutor, CommitHandler commit) { super(databaseConfig); this.storage = storage; @@ -128,7 +134,7 @@ public TwoPhaseConsensusCommitManager(DatabaseConfig databaseConfig) { admin, databaseConfig.getMetadataCacheExpirationTimeSecs()); this.coordinator = coordinator; this.parallelExecutor = parallelExecutor; - this.recovery = recovery; + this.recoveryExecutor = recoveryExecutor; this.commit = commit; isIncludeMetadataEnabled = config.isIncludeMetadataEnabled(); mutationOperationChecker = new ConsensusCommitMutationOperationChecker(tableMetadataManager); @@ -185,13 +191,14 @@ TwoPhaseCommitTransaction begin( new CrudHandler( storage, snapshot, + recoveryExecutor, tableMetadataManager, isIncludeMetadataEnabled, parallelExecutor, readOnly, oneOperation); TwoPhaseConsensusCommit transaction = - new TwoPhaseConsensusCommit(crud, commit, recovery, mutationOperationChecker); + new TwoPhaseConsensusCommit(crud, commit, mutationOperationChecker); getNamespace().ifPresent(transaction::withNamespace); getTable().ifPresent(transaction::withTable); return transaction; @@ -449,5 +456,6 @@ public void close() { storage.close(); admin.close(); parallelExecutor.close(); + recoveryExecutor.close(); } } diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfigTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfigTest.java index d93fd0ad8a..c9cb63eaf9 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfigTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfigTest.java @@ -27,8 +27,9 @@ public void constructor_NoPropertiesGiven_ShouldLoadAsDefaultValues() { assertThat(config.isParallelRollbackEnabled()).isTrue(); assertThat(config.isAsyncCommitEnabled()).isFalse(); assertThat(config.isAsyncRollbackEnabled()).isFalse(); - assertThat(config.isParallelImplicitPreReadEnabled()).isTrue(); assertThat(config.isCoordinatorWriteOmissionOnReadOnlyEnabled()).isTrue(); + assertThat(config.getRecoveryExecutorCount()).isEqualTo(128); + assertThat(config.isParallelImplicitPreReadEnabled()).isTrue(); assertThat(config.isIncludeMetadataEnabled()).isFalse(); } @@ -159,6 +160,19 @@ public void constructor_AsyncExecutionRelatedPropertiesGiven_ShouldLoadProperly( constructor_PropertiesWithCoordinatorWriteOmissionOnReadOnlyEnabledGiven_ShouldLoadProperly() { // Arrange Properties props = new Properties(); + props.setProperty(ConsensusCommitConfig.RECOVERY_EXECUTOR_COUNT, "256"); + + // Act + ConsensusCommitConfig config = new ConsensusCommitConfig(new DatabaseConfig(props)); + + // Assert + assertThat(config.getRecoveryExecutorCount()).isEqualTo(256); + } + + @Test + public void constructor_PropertiesWithRecoveryExecutorCountGiven_ShouldLoadProperly() { + // Arrange + Properties props = new Properties(); props.setProperty( ConsensusCommitConfig.COORDINATOR_WRITE_OMISSION_ON_READ_ONLY_ENABLED, "false"); diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManagerTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManagerTest.java index 3cb717a846..77152177fa 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManagerTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManagerTest.java @@ -63,7 +63,7 @@ public class ConsensusCommitManagerTest { @Mock private ConsensusCommitConfig consensusCommitConfig; @Mock private Coordinator coordinator; @Mock private ParallelExecutor parallelExecutor; - @Mock private RecoveryHandler recovery; + @Mock private RecoveryExecutor recoveryExecutor; @Mock private CommitHandler commit; private ConsensusCommitManager manager; @@ -80,7 +80,7 @@ public void setUp() throws Exception { databaseConfig, coordinator, parallelExecutor, - recovery, + recoveryExecutor, commit, null); @@ -131,7 +131,7 @@ public void begin_TxIdGiven_ReturnWithSpecifiedTxIdAndSnapshotIsolation() { databaseConfig, coordinator, parallelExecutor, - recovery, + recoveryExecutor, commit, groupCommitter); @@ -162,7 +162,7 @@ public void begin_TxIdGiven_ReturnWithSpecifiedTxIdAndSnapshotIsolation() { databaseConfig, coordinator, parallelExecutor, - recovery, + recoveryExecutor, commit, groupCommitter); @@ -196,9 +196,6 @@ public void begin_CalledTwice_ReturnRespectiveConsensusCommitWithSharedCommitAnd assertThat(transaction1.getCommitHandler()) .isEqualTo(transaction2.getCommitHandler()) .isEqualTo(commit); - assertThat(transaction1.getRecoveryHandler()) - .isEqualTo(transaction2.getRecoveryHandler()) - .isEqualTo(recovery); } @Test @@ -309,9 +306,6 @@ public void start_CalledTwice_ReturnRespectiveConsensusCommitWithSharedCommitAnd assertThat(transaction1.getCommitHandler()) .isEqualTo(transaction2.getCommitHandler()) .isEqualTo(commit); - assertThat(transaction1.getRecoveryHandler()) - .isEqualTo(transaction2.getRecoveryHandler()) - .isEqualTo(recovery); } @Test diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitTest.java index 99a6f6ae37..95b837a748 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitTest.java @@ -57,7 +57,6 @@ public class ConsensusCommitTest { @Mock private Snapshot snapshot; @Mock private CrudHandler crud; @Mock private CommitHandler commit; - @Mock private RecoveryHandler recovery; @SuppressWarnings("unused") @Mock @@ -117,26 +116,9 @@ public void get_GetGiven_ShouldCallCrudHandlerGet() throws CrudException { // Assert assertThat(actual).isPresent(); - verify(recovery, never()).recover(get, result); verify(crud).get(get); } - @Test - public void get_GetForUncommittedRecordGiven_ShouldRecoverRecord() throws CrudException { - // Arrange - Get get = prepareGet(); - TransactionResult result = mock(TransactionResult.class); - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - when(crud.get(get)).thenThrow(toThrow); - when(toThrow.getSelection()).thenReturn(get); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - // Act Assert - assertThatThrownBy(() -> consensus.get(get)).isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(get, result); - } - @Test public void scan_ScanGiven_ShouldCallCrudHandlerScan() throws CrudException { // Arrange @@ -153,22 +135,6 @@ public void scan_ScanGiven_ShouldCallCrudHandlerScan() throws CrudException { verify(crud).scan(scan); } - @Test - public void scan_ScanForUncommittedRecordGiven_ShouldRecoverRecord() throws CrudException { - // Arrange - Scan scan = prepareScan(); - TransactionResult result = mock(TransactionResult.class); - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - when(crud.scan(scan)).thenThrow(toThrow); - when(toThrow.getSelection()).thenReturn(scan); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - // Act Assert - assertThatThrownBy(() -> consensus.scan(scan)).isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(scan, result); - } - @Test public void getScannerAndScannerOne_ShouldCallCrudHandlerGetScannerAndScannerOne() throws CrudException { @@ -189,29 +155,6 @@ public void getScannerAndScannerOne_ShouldCallCrudHandlerGetScannerAndScannerOne verify(scanner).one(); } - @Test - public void - getScannerAndScannerOne_UncommittedRecordExceptionThrownByScannerOne_ShouldRecoverRecord() - throws CrudException { - // Arrange - Scan scan = prepareScan(); - - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - TransactionResult result = mock(TransactionResult.class); - when(toThrow.getSelection()).thenReturn(scan); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); - when(scanner.one()).thenThrow(toThrow); - when(crud.getScanner(scan)).thenReturn(scanner); - - // Act Assert - TransactionCrudOperable.Scanner actualScanner = consensus.getScanner(scan); - assertThatThrownBy(actualScanner::one).isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(scan, result); - } - @Test public void getScannerAndScannerAll_ShouldCallCrudHandlerGetScannerAndScannerAll() throws CrudException { @@ -233,29 +176,6 @@ public void getScannerAndScannerAll_ShouldCallCrudHandlerGetScannerAndScannerAll verify(scanner).all(); } - @Test - public void - getScannerAndScannerAll_UncommittedRecordExceptionThrownByScannerAll_ShouldRecoverRecord() - throws CrudException { - // Arrange - Scan scan = prepareScan(); - - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - TransactionResult result = mock(TransactionResult.class); - when(toThrow.getSelection()).thenReturn(scan); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); - when(scanner.all()).thenThrow(toThrow); - when(crud.getScanner(scan)).thenReturn(scanner); - - // Act Assert - TransactionCrudOperable.Scanner actualScanner = consensus.getScanner(scan); - assertThatThrownBy(actualScanner::all).isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(scan, result); - } - @Test public void put_PutGiven_ShouldCallCrudHandlerPut() throws ExecutionException, CrudException { // Arrange @@ -285,25 +205,6 @@ public void put_TwoPutsGiven_ShouldCallCrudHandlerPutTwice() verify(mutationOperationChecker, times(2)).check(put); } - @Test - public void put_PutGivenAndUncommittedRecordExceptionThrown_ShouldRecoverRecord() - throws CrudException { - // Arrange - Put put = preparePut(); - Get get = prepareGet(); - - TransactionResult result = mock(TransactionResult.class); - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - doThrow(toThrow).when(crud).put(put); - when(toThrow.getSelection()).thenReturn(get); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - // Act Assert - assertThatThrownBy(() -> consensus.put(put)).isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(get, result); - } - @Test public void delete_DeleteGiven_ShouldCallCrudHandlerDelete() throws CrudException, ExecutionException { @@ -334,26 +235,6 @@ public void delete_TwoDeletesGiven_ShouldCallCrudHandlerDeleteTwice() verify(mutationOperationChecker, times(2)).check(delete); } - @Test - public void delete_DeleteGivenAndUncommittedRecordExceptionThrown_ShouldRecoverRecord() - throws CrudException { - // Arrange - Delete delete = prepareDelete(); - Get get = prepareGet(); - - TransactionResult result = mock(TransactionResult.class); - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - doThrow(toThrow).when(crud).delete(delete); - when(toThrow.getSelection()).thenReturn(get); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - // Act Assert - assertThatThrownBy(() -> consensus.delete(delete)) - .isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(get, result); - } - @Test public void insert_InsertGiven_ShouldCallCrudHandlerPut() throws CrudException, ExecutionException { @@ -414,47 +295,6 @@ public void upsert_UpsertGiven_ShouldCallCrudHandlerPut() verify(mutationOperationChecker).check(expectedPut); } - @Test - public void upsert_UpsertForUncommittedRecordGiven_ShouldRecoverRecord() throws CrudException { - // Arrange - Upsert upsert = - Upsert.newBuilder() - .namespace(ANY_NAMESPACE) - .table(ANY_TABLE_NAME) - .partitionKey(Key.ofText(ANY_NAME_1, ANY_TEXT_1)) - .clusteringKey(Key.ofText(ANY_NAME_2, ANY_TEXT_2)) - .textValue(ANY_NAME_3, ANY_TEXT_3) - .build(); - Put put = - Put.newBuilder() - .namespace(ANY_NAMESPACE) - .table(ANY_TABLE_NAME) - .partitionKey(Key.ofText(ANY_NAME_1, ANY_TEXT_1)) - .clusteringKey(Key.ofText(ANY_NAME_2, ANY_TEXT_2)) - .textValue(ANY_NAME_3, ANY_TEXT_3) - .enableImplicitPreRead() - .build(); - Get get = - Get.newBuilder() - .namespace(ANY_NAMESPACE) - .table(ANY_TABLE_NAME) - .partitionKey(Key.ofText(ANY_NAME_1, ANY_TEXT_1)) - .clusteringKey(Key.ofText(ANY_NAME_2, ANY_TEXT_2)) - .build(); - - TransactionResult result = mock(TransactionResult.class); - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - doThrow(toThrow).when(crud).put(put); - when(toThrow.getSelection()).thenReturn(get); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - // Act Assert - assertThatThrownBy(() -> consensus.upsert(upsert)) - .isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(get, result); - } - @Test public void update_UpdateWithoutConditionGiven_ShouldCallCrudHandlerPut() throws CrudException, ExecutionException { @@ -643,48 +483,6 @@ public void update_UpdateWithConditionGiven_ShouldCallCrudHandlerPut() .hasMessageNotContaining("PutIfExists"); } - @Test - public void update_UpdateForUncommittedRecordGiven_ShouldRecoverRecord() throws CrudException { - // Arrange - Update update = - Update.newBuilder() - .namespace(ANY_NAMESPACE) - .table(ANY_TABLE_NAME) - .partitionKey(Key.ofText(ANY_NAME_1, ANY_TEXT_1)) - .clusteringKey(Key.ofText(ANY_NAME_2, ANY_TEXT_2)) - .textValue(ANY_NAME_3, ANY_TEXT_3) - .build(); - Put put = - Put.newBuilder() - .namespace(ANY_NAMESPACE) - .table(ANY_TABLE_NAME) - .partitionKey(Key.ofText(ANY_NAME_1, ANY_TEXT_1)) - .clusteringKey(Key.ofText(ANY_NAME_2, ANY_TEXT_2)) - .textValue(ANY_NAME_3, ANY_TEXT_3) - .condition(ConditionBuilder.putIfExists()) - .enableImplicitPreRead() - .build(); - Get get = - Get.newBuilder() - .namespace(ANY_NAMESPACE) - .table(ANY_TABLE_NAME) - .partitionKey(Key.ofText(ANY_NAME_1, ANY_TEXT_1)) - .clusteringKey(Key.ofText(ANY_NAME_2, ANY_TEXT_2)) - .build(); - - TransactionResult result = mock(TransactionResult.class); - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - doThrow(toThrow).when(crud).put(put); - when(toThrow.getSelection()).thenReturn(get); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - // Act Assert - assertThatThrownBy(() -> consensus.update(update)) - .isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(get, result); - } - @Test public void mutate_PutAndDeleteGiven_ShouldCallCrudHandlerPutAndDelete() throws CrudException, ExecutionException { @@ -716,7 +514,9 @@ public void commit_ProcessedCrudGiven_ShouldCommitWithSnapshot() consensus.commit(); // Assert + verify(crud).areAllScannersClosed(); verify(crud).readIfImplicitPreReadEnabled(); + verify(crud).waitForRecoveryCompletionIfNecessary(); verify(commit).commit(snapshot, false); } @@ -732,7 +532,9 @@ public void commit_ProcessedCrudGiven_InReadOnlyMode_ShouldCommitWithSnapshot() consensus.commit(); // Assert + verify(crud).areAllScannersClosed(); verify(crud).readIfImplicitPreReadEnabled(); + verify(crud).waitForRecoveryCompletionIfNecessary(); verify(commit).commit(snapshot, true); } @@ -748,28 +550,6 @@ public void commit_ProcessedCrudGiven_InReadOnlyMode_ShouldCommitWithSnapshot() assertThatThrownBy(() -> consensus.commit()).isInstanceOf(CommitConflictException.class); } - @Test - public void - commit_ProcessedCrudGiven_UncommittedRecordExceptionThrownWhileImplicitPreRead_ShouldPerformLazyRecoveryAndThrowCommitConflictException() - throws CrudException { - // Arrange - when(crud.getSnapshot()).thenReturn(snapshot); - - Get get = mock(Get.class); - TransactionResult result = mock(TransactionResult.class); - - UncommittedRecordException uncommittedRecordException = mock(UncommittedRecordException.class); - when(uncommittedRecordException.getSelection()).thenReturn(get); - when(uncommittedRecordException.getResults()).thenReturn(Collections.singletonList(result)); - - doThrow(uncommittedRecordException).when(crud).readIfImplicitPreReadEnabled(); - - // Act Assert - assertThatThrownBy(() -> consensus.commit()).isInstanceOf(CommitConflictException.class); - - verify(recovery).recover(get, result); - } - @Test public void commit_ProcessedCrudGiven_CrudExceptionThrownWhileImplicitPreRead_ShouldThrowCommitException() @@ -791,6 +571,18 @@ public void commit_ScannerNotClosed_ShouldThrowIllegalStateException() { assertThatThrownBy(() -> consensus.commit()).isInstanceOf(IllegalStateException.class); } + @Test + public void + commit_CrudExceptionThrownByCrudHandlerWaitForRecoveryCompletionIfNecessary_ShouldThrowCommitException() + throws CrudException { + // Arrange + when(crud.getSnapshot()).thenReturn(snapshot); + doThrow(CrudException.class).when(crud).waitForRecoveryCompletionIfNecessary(); + + // Act Assert + assertThatThrownBy(() -> consensus.commit()).isInstanceOf(CommitException.class); + } + @Test public void rollback_ShouldDoNothing() throws CrudException, UnknownTransactionStatusException { // Arrange @@ -814,7 +606,7 @@ public void rollback_WithGroupCommitter_ShouldRemoveTxFromGroupCommitter() doReturn(snapshot).when(crud).getSnapshot(); CoordinatorGroupCommitter groupCommitter = mock(CoordinatorGroupCommitter.class); ConsensusCommit consensusWithGroupCommit = - new ConsensusCommit(crud, commit, recovery, mutationOperationChecker, groupCommitter); + new ConsensusCommit(crud, commit, mutationOperationChecker, groupCommitter); // Act consensusWithGroupCommit.rollback(); @@ -837,7 +629,7 @@ public void rollback_WithGroupCommitter_InReadOnlyMode_ShouldNotRemoveTxFromGrou doReturn(true).when(crud).isReadOnly(); CoordinatorGroupCommitter groupCommitter = mock(CoordinatorGroupCommitter.class); ConsensusCommit consensusWithGroupCommit = - new ConsensusCommit(crud, commit, recovery, mutationOperationChecker, groupCommitter); + new ConsensusCommit(crud, commit, mutationOperationChecker, groupCommitter); // Act consensusWithGroupCommit.rollback(); diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtilsTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtilsTest.java index 3bc9cd716f..1eb8b6565f 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtilsTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtilsTest.java @@ -2,9 +2,18 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.scalar.db.api.TableMetadata; +import com.scalar.db.io.Column; import com.scalar.db.io.DataType; +import com.scalar.db.io.IntColumn; +import com.scalar.db.io.TextColumn; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import org.junit.jupiter.api.Test; public class ConsensusCommitUtilsTest { @@ -614,4 +623,81 @@ void getNextTxVersion_LargeValueGiven_shouldReturnNextVersion() { // Act Assert assertThat(ConsensusCommitUtils.getNextTxVersion(100000)).isEqualTo(100001); } + + @Test + public void extractAfterImageColumnsFromBeforeImage_shouldExtractCorrectly() { + // Arrange + Map> columns = new HashMap<>(); + Set beforeImageColumnNames = new HashSet<>(); + beforeImageColumnNames.add("before_balance"); + beforeImageColumnNames.add("before_name"); + + Map> resultColumns = new HashMap<>(); + resultColumns.put("before_balance", IntColumn.of("before_balance", 1000)); + resultColumns.put("before_name", TextColumn.of("before_name", "Alice")); + resultColumns.put("account_id", IntColumn.of("account_id", 123)); // Not a before column + + TransactionResult result = mock(TransactionResult.class); + when(result.getColumns()).thenReturn(resultColumns); + + // Act + ConsensusCommitUtils.extractAfterImageColumnsFromBeforeImage( + columns, result, beforeImageColumnNames); + + // Assert + assertThat(columns).hasSize(2); + assertThat(columns).containsKey("balance"); + assertThat(columns).containsKey("name"); + assertThat(columns.get("balance").getIntValue()).isEqualTo(1000); + assertThat(columns.get("name").getTextValue()).isEqualTo("Alice"); + assertThat(columns).doesNotContainKey("account_id"); + } + + @Test + public void + extractAfterImageColumnsFromBeforeImage_versionColumnWithZero_shouldCreateNullVersion() { + // Arrange + Map> columns = new HashMap<>(); + Set beforeImageColumnNames = new HashSet<>(); + beforeImageColumnNames.add("before_tx_version"); + + Map> resultColumns = new HashMap<>(); + resultColumns.put("before_tx_version", IntColumn.of("before_tx_version", 0)); + + TransactionResult result = mock(TransactionResult.class); + when(result.getColumns()).thenReturn(resultColumns); + + // Act + ConsensusCommitUtils.extractAfterImageColumnsFromBeforeImage( + columns, result, beforeImageColumnNames); + + // Assert + assertThat(columns).hasSize(1); + assertThat(columns).containsKey("tx_version"); + assertThat(columns.get("tx_version").hasNullValue()).isTrue(); + } + + @Test + public void extractAfterImageColumnsFromBeforeImage_versionColumnWithNonZero_shouldCopyValue() { + // Arrange + Map> columns = new HashMap<>(); + Set beforeImageColumnNames = new HashSet<>(); + beforeImageColumnNames.add("before_tx_version"); + + Map> resultColumns = new HashMap<>(); + resultColumns.put("before_tx_version", IntColumn.of("before_tx_version", 5)); + + TransactionResult result = mock(TransactionResult.class); + when(result.getColumns()).thenReturn(resultColumns); + + // Act + ConsensusCommitUtils.extractAfterImageColumnsFromBeforeImage( + columns, result, beforeImageColumnNames); + + // Assert + assertThat(columns).hasSize(1); + assertThat(columns).containsKey("tx_version"); + assertThat(columns.get("tx_version").hasNullValue()).isFalse(); + assertThat(columns.get("tx_version").getIntValue()).isEqualTo(5); + } } diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/CrudHandlerTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/CrudHandlerTest.java index 5fbf1bf9a5..0302522112 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/CrudHandlerTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/CrudHandlerTest.java @@ -52,6 +52,7 @@ import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Future; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -87,12 +88,11 @@ public class CrudHandlerTest { .addClusteringKey(ANY_NAME_2) .addSecondaryIndex(ANY_NAME_3) .build()); - private static final TransactionTableMetadata TRANSACTION_TABLE_METADATA = - new TransactionTableMetadata(TABLE_METADATA); private CrudHandler handler; @Mock private DistributedStorage storage; @Mock private Snapshot snapshot; + @Mock private RecoveryExecutor recoveryExecutor; @Mock private TransactionTableMetadataManager tableMetadataManager; @Mock private ParallelExecutor parallelExecutor; @Mock private Scanner scanner; @@ -106,6 +106,7 @@ public void setUp() throws Exception { new CrudHandler( storage, snapshot, + recoveryExecutor, tableMetadataManager, false, mutationConditionsValidator, @@ -129,11 +130,7 @@ private Get prepareGet() { } private Get toGetForStorageFrom(Get get) { - return Get.newBuilder(get) - .clearProjections() - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) - .consistency(Consistency.LINEARIZABLE) - .build(); + return Get.newBuilder(get).clearProjections().consistency(Consistency.LINEARIZABLE).build(); } private Scan prepareScan() { @@ -151,11 +148,7 @@ private Scan prepareCrossPartitionScan() { } private Scan toScanForStorageFrom(Scan scan) { - return Scan.newBuilder(scan) - .clearProjections() - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) - .consistency(Consistency.LINEARIZABLE) - .build(); + return Scan.newBuilder(scan).clearProjections().consistency(Consistency.LINEARIZABLE).build(); } private TransactionResult prepareResult(TransactionState state) { @@ -240,6 +233,7 @@ public void get_GetExistsInSnapshot_ShouldReturnFromSnapshot() throws CrudExcept new CrudHandler( storage, snapshot, + recoveryExecutor, tableMetadataManager, false, mutationConditionsValidator, @@ -279,6 +273,7 @@ public void get_GetExistsInSnapshot_ShouldReturnFromSnapshot() throws CrudExcept new CrudHandler( storage, snapshot, + recoveryExecutor, tableMetadataManager, false, mutationConditionsValidator, @@ -320,6 +315,7 @@ public void get_GetExistsInSnapshot_ShouldReturnFromSnapshot() throws CrudExcept new CrudHandler( storage, snapshot, + recoveryExecutor, tableMetadataManager, false, mutationConditionsValidator, @@ -361,6 +357,7 @@ public void get_GetExistsInSnapshot_ShouldReturnFromSnapshot() throws CrudExcept new CrudHandler( storage, snapshot, + recoveryExecutor, tableMetadataManager, false, mutationConditionsValidator, @@ -403,30 +400,43 @@ public void get_GetExistsInSnapshot_ShouldReturnFromSnapshot() throws CrudExcept } @Test - public void - get_GetNotExistsInSnapshotAndRecordInStorageNotCommitted_ShouldThrowUncommittedRecordException() - throws ExecutionException { + public void get_GetNotExistsInSnapshotAndRecordInStorageNotCommitted_ShouldCallRecoveryExecutor() + throws ExecutionException, CrudException { // Arrange Get get = prepareGet(); + Snapshot.Key key = new Snapshot.Key(get); Get getForStorage = toGetForStorageFrom(get); result = prepareResult(TransactionState.PREPARED); - Optional expected = Optional.of(result); + when(storage.get(getForStorage)).thenReturn(Optional.of(result)); when(snapshot.containsKeyInGetSet(getForStorage)).thenReturn(false); - when(storage.get(getForStorage)).thenReturn(expected); + when(snapshot.getId()).thenReturn(ANY_ID_1); - // Act Assert - assertThatThrownBy(() -> handler.get(get)) - .isInstanceOf(UncommittedRecordException.class) - .satisfies( - e -> { - UncommittedRecordException exception = (UncommittedRecordException) e; - assertThat(exception.getSelection()).isEqualTo(get); - assertThat(exception.getResults().size()).isEqualTo(1); - assertThat(exception.getResults().get(0)).isEqualTo(result); - }); + TransactionResult expected = mock(TransactionResult.class); + when(expected.getContainedColumnNames()).thenReturn(Collections.singleton(ANY_NAME_1)); + when(expected.getAsObject(ANY_NAME_1)).thenReturn(ANY_TEXT_1); - verify(snapshot, never()).putIntoReadSet(any(), any()); - verify(snapshot, never()).putIntoGetSet(any(), any()); + when(snapshot.getResult(key, getForStorage)).thenReturn(Optional.of(expected)); + + TransactionResult recoveredResult = mock(TransactionResult.class); + @SuppressWarnings("unchecked") + Future recoveryFuture = mock(Future.class); + + when(recoveryExecutor.execute(key, getForStorage, new TransactionResult(result), ANY_ID_1)) + .thenReturn(new RecoveryExecutor.Result(key, Optional.of(recoveredResult), recoveryFuture)); + + // Act + Optional actual = handler.get(get); + + // Assert + verify(storage).get(getForStorage); + verify(recoveryExecutor).execute(key, getForStorage, new TransactionResult(result), ANY_ID_1); + verify(snapshot).putIntoReadSet(key, Optional.of(recoveredResult)); + verify(snapshot).putIntoGetSet(getForStorage, Optional.of(recoveredResult)); + + assertThat(actual) + .isEqualTo( + Optional.of( + new FilteredResult(expected, Collections.emptyList(), TABLE_METADATA, false))); } @Test @@ -503,7 +513,14 @@ public void get_CalledTwiceUnderRealSnapshot_SecondTimeShouldReturnTheSameFromSn snapshot = new Snapshot(ANY_TX_ID, Isolation.SNAPSHOT, tableMetadataManager, parallelExecutor); handler = new CrudHandler( - storage, snapshot, tableMetadataManager, false, parallelExecutor, false, false); + storage, + snapshot, + recoveryExecutor, + tableMetadataManager, + false, + parallelExecutor, + false, + false); when(storage.get(getForStorage)).thenReturn(Optional.of(result)); // Act @@ -580,6 +597,7 @@ void scanOrGetScanner_ResultGivenFromStorage_InReadOnlyMode_ShouldUpdateSnapshot new CrudHandler( storage, snapshot, + recoveryExecutor, tableMetadataManager, false, mutationConditionsValidator, @@ -621,6 +639,7 @@ void scanOrGetScanner_ResultGivenFromStorage_InReadOnlyMode_ShouldUpdateSnapshot new CrudHandler( storage, snapshot, + recoveryExecutor, tableMetadataManager, false, mutationConditionsValidator, @@ -662,6 +681,7 @@ void scanOrGetScanner_ResultGivenFromStorage_InReadOnlyMode_ShouldUpdateSnapshot new CrudHandler( storage, snapshot, + recoveryExecutor, tableMetadataManager, false, mutationConditionsValidator, @@ -696,12 +716,12 @@ void scanOrGetScanner_ResultGivenFromStorage_InReadOnlyMode_ShouldUpdateSnapshot @ParameterizedTest @EnumSource(ScanType.class) - void - scanOrGetScanner_PreparedResultGivenFromStorage_ShouldNeverUpdateSnapshotThrowUncommittedRecordException( - ScanType scanType) throws ExecutionException, IOException { + void scanOrGetScanner_PreparedResultGivenFromStorage_ShouldCallRecoveryExecutor(ScanType scanType) + throws ExecutionException, IOException, CrudException { // Arrange Scan scan = prepareScan(); Scan scanForStorage = toScanForStorageFrom(scan); + result = prepareResult(TransactionState.PREPARED); if (scanType == ScanType.SCAN) { when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); @@ -710,20 +730,34 @@ void scanOrGetScanner_ResultGivenFromStorage_InReadOnlyMode_ShouldUpdateSnapshot } when(storage.scan(scanForStorage)).thenReturn(scanner); - // Act Assert - assertThatThrownBy(() -> scanOrGetScanner(scan, scanType)) - .isInstanceOf(UncommittedRecordException.class) - .satisfies( - e -> { - UncommittedRecordException exception = (UncommittedRecordException) e; - assertThat(exception.getSelection()).isEqualTo(scan); - assertThat(exception.getResults().size()).isEqualTo(1); - assertThat(exception.getResults().get(0)).isEqualTo(result); - }); + Snapshot.Key key = new Snapshot.Key(scan, result); + + when(snapshot.getId()).thenReturn(ANY_ID_1); + + TransactionResult recoveredResult = mock(TransactionResult.class); + when(recoveredResult.getContainedColumnNames()).thenReturn(Collections.singleton(ANY_NAME_1)); + when(recoveredResult.getAsObject(ANY_NAME_1)).thenReturn(ANY_TEXT_1); + @SuppressWarnings("unchecked") + Future recoveryFuture = mock(Future.class); + + when(recoveryExecutor.execute(key, scanForStorage, new TransactionResult(result), ANY_ID_1)) + .thenReturn(new RecoveryExecutor.Result(key, Optional.of(recoveredResult), recoveryFuture)); + + // Act + List results = scanOrGetScanner(scan, scanType); + + // Assert verify(scanner).close(); - verify(snapshot, never()).putIntoReadSet(any(), any()); - verify(snapshot, never()).putIntoScanSet(any(), any()); + verify(snapshot).putIntoReadSet(key, Optional.of(recoveredResult)); + verify(snapshot) + .putIntoScanSet( + scanForStorage, Maps.newLinkedHashMap(ImmutableMap.of(key, recoveredResult))); + verify(snapshot).verifyNoOverlap(scanForStorage, ImmutableMap.of(key, recoveredResult)); + + assertThat(results) + .containsExactly( + new FilteredResult(recoveredResult, Collections.emptyList(), TABLE_METADATA, false)); } @ParameterizedTest @@ -779,7 +813,14 @@ void scan_CalledTwiceUnderRealSnapshot_SecondTimeShouldReturnTheSameFromSnapshot snapshot = new Snapshot(ANY_TX_ID, Isolation.SNAPSHOT, tableMetadataManager, parallelExecutor); handler = new CrudHandler( - storage, snapshot, tableMetadataManager, false, parallelExecutor, false, false); + storage, + snapshot, + recoveryExecutor, + tableMetadataManager, + false, + parallelExecutor, + false, + false); if (scanType == ScanType.SCAN) { when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); } else { @@ -848,7 +889,14 @@ void scanOrGetScanner_GetCalledAfterScanUnderRealSnapshot_ShouldReturnFromStorag snapshot = new Snapshot(ANY_TX_ID, Isolation.SNAPSHOT, tableMetadataManager, parallelExecutor); handler = new CrudHandler( - storage, snapshot, tableMetadataManager, false, parallelExecutor, false, false); + storage, + snapshot, + recoveryExecutor, + tableMetadataManager, + false, + parallelExecutor, + false, + false); if (scanType == ScanType.SCAN) { when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); } else { @@ -915,7 +963,14 @@ void scanOrGetScanner_CalledAfterDeleteUnderRealSnapshot_ShouldThrowIllegalArgum new ArrayList<>()); handler = new CrudHandler( - storage, snapshot, tableMetadataManager, false, parallelExecutor, false, false); + storage, + snapshot, + recoveryExecutor, + tableMetadataManager, + false, + parallelExecutor, + false, + false); if (scanType == ScanType.SCAN) { when(scanner.iterator()).thenReturn(Arrays.asList(result, result2).iterator()); } else { @@ -979,11 +1034,15 @@ void scanOrGetScanner_CalledAfterDeleteUnderRealSnapshot_ShouldThrowIllegalArgum @ParameterizedTest @EnumSource(ScanType.class) void - scanOrGetScanner_CrossPartitionScanAndPreparedResultFromStorageGiven_ShouldNeverUpdateSnapshotNorVerifyNoOverlapButThrowUncommittedRecordException( - ScanType scanType) throws ExecutionException, IOException { + scanOrGetScanner_CrossPartitionScanAndPreparedResultFromStorageGiven_RecoveredRecordMatchesConjunction_ShouldCallRecoveryExecutor( + ScanType scanType) throws ExecutionException, IOException, CrudException { // Arrange Scan scan = prepareCrossPartitionScan(); + Scan scanForStorage = toScanForStorageFrom(scan); + result = prepareResult(TransactionState.PREPARED); + Snapshot.Key key = new Snapshot.Key(scanForStorage, result); + when(snapshot.getId()).thenReturn(ANY_ID_1); if (scanType == ScanType.SCAN) { when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); } else { @@ -991,21 +1050,93 @@ void scanOrGetScanner_CalledAfterDeleteUnderRealSnapshot_ShouldThrowIllegalArgum } when(storage.scan(any(ScanAll.class))).thenReturn(scanner); - // Act Assert - assertThatThrownBy(() -> scanOrGetScanner(scan, scanType)) - .isInstanceOf(UncommittedRecordException.class) - .satisfies( - e -> { - UncommittedRecordException exception = (UncommittedRecordException) e; - assertThat(exception.getSelection()).isEqualTo(scan); - assertThat(exception.getResults().size()).isEqualTo(1); - assertThat(exception.getResults().get(0)).isEqualTo(result); - }); + TransactionResult recoveredResult = mock(TransactionResult.class); + when(recoveredResult.getContainedColumnNames()).thenReturn(Collections.singleton(ANY_NAME_3)); + when(recoveredResult.getAsObject(ANY_NAME_3)).thenReturn(ANY_TEXT_3); + when(recoveredResult.getColumns()) + .thenReturn(ImmutableMap.of(ANY_NAME_3, TextColumn.of(ANY_NAME_3, ANY_TEXT_3))); + when(snapshot.getResult(key)).thenReturn(Optional.of(new TransactionResult(result))); + + @SuppressWarnings("unchecked") + Future recoveryFuture = mock(Future.class); + + when(recoveryExecutor.execute(key, scanForStorage, new TransactionResult(result), ANY_ID_1)) + .thenReturn(new RecoveryExecutor.Result(key, Optional.of(recoveredResult), recoveryFuture)); + + // Act + List results = scanOrGetScanner(scanForStorage, scanType); + + // Assert + verify(storage) + .scan( + Scan.newBuilder(scanForStorage) + .clearConditions() + .where(column(ANY_NAME_3).isEqualToText(ANY_TEXT_3)) + .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isEqualToText(ANY_TEXT_3)) + .build()); verify(scanner).close(); - verify(snapshot, never()).putIntoReadSet(any(Snapshot.Key.class), any()); - verify(snapshot, never()).putIntoScannerSet(any(Scan.class), any()); - verify(snapshot, never()).verifyNoOverlap(any(), any()); + verify(snapshot).putIntoReadSet(key, Optional.of(recoveredResult)); + verify(snapshot) + .putIntoScanSet( + scanForStorage, Maps.newLinkedHashMap(ImmutableMap.of(key, recoveredResult))); + verify(snapshot).verifyNoOverlap(scanForStorage, ImmutableMap.of(key, recoveredResult)); + + assertThat(results) + .containsExactly( + new FilteredResult(recoveredResult, Collections.emptyList(), TABLE_METADATA, false)); + } + + @ParameterizedTest + @EnumSource(ScanType.class) + void + scanOrGetScanner_CrossPartitionScanAndPreparedResultFromStorageGiven_RecoveredRecordDoesNotMatchConjunction_ShouldCallRecoveryExecutor( + ScanType scanType) throws ExecutionException, IOException, CrudException { + // Arrange + Scan scan = prepareCrossPartitionScan(); + Scan scanForStorage = toScanForStorageFrom(scan); + + result = prepareResult(TransactionState.PREPARED); + Snapshot.Key key = new Snapshot.Key(scanForStorage, result); + when(snapshot.getId()).thenReturn(ANY_ID_1); + if (scanType == ScanType.SCAN) { + when(scanner.iterator()).thenReturn(Collections.singletonList(result).iterator()); + } else { + when(scanner.one()).thenReturn(Optional.of(result)).thenReturn(Optional.empty()); + } + when(storage.scan(any(ScanAll.class))).thenReturn(scanner); + + TransactionResult recoveredResult = mock(TransactionResult.class); + when(recoveredResult.getContainedColumnNames()).thenReturn(Collections.singleton(ANY_NAME_3)); + when(recoveredResult.getAsObject(ANY_NAME_3)).thenReturn(ANY_TEXT_4); + when(recoveredResult.getColumns()) + .thenReturn(ImmutableMap.of(ANY_NAME_3, TextColumn.of(ANY_NAME_3, ANY_TEXT_4))); + + when(snapshot.getResult(key)).thenReturn(Optional.of(new TransactionResult(result))); + + @SuppressWarnings("unchecked") + Future recoveryFuture = mock(Future.class); + + when(recoveryExecutor.execute(key, scanForStorage, new TransactionResult(result), ANY_ID_1)) + .thenReturn(new RecoveryExecutor.Result(key, Optional.of(recoveredResult), recoveryFuture)); + + // Act + List results = scanOrGetScanner(scanForStorage, scanType); + + // Assert + verify(storage) + .scan( + Scan.newBuilder(scanForStorage) + .clearConditions() + .where(column(ANY_NAME_3).isEqualToText(ANY_TEXT_3)) + .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isEqualToText(ANY_TEXT_3)) + .build()); + verify(scanner).close(); + verify(snapshot, never()).putIntoReadSet(any(), any()); + verify(snapshot).putIntoScanSet(scanForStorage, Maps.newLinkedHashMap()); + verify(snapshot).verifyNoOverlap(scanForStorage, ImmutableMap.of()); + + assertThat(results).isEmpty(); } @ParameterizedTest @@ -1096,30 +1227,86 @@ void scanOrGetScanner_WithLimitExceedingAvailableResults_ShouldReturnAllAvailabl @ParameterizedTest @EnumSource(ScanType.class) - void scanOrGetScanner_WithLimit_UncommittedResult_ShouldThrowUncommittedRecordException( - ScanType scanType) throws ExecutionException, IOException { + void scanOrGetScanner_WithLimit_UncommittedResult_ShouldCallRecoveryExecutor(ScanType scanType) + throws ExecutionException, IOException, CrudException { // Arrange Scan scanWithoutLimit = prepareScan(); - Scan scanWithLimit = Scan.newBuilder(scanWithoutLimit).limit(3).build(); - Scan scanForStorage = toScanForStorageFrom(scanWithoutLimit); + Scan scanWithLimit = Scan.newBuilder(scanWithoutLimit).limit(2).build(); + Scan scanForStorageWithLimit = toScanForStorageFrom(scanWithLimit); + Scan scanForStorageWithoutLimit = toScanForStorageFrom(scanWithoutLimit); + + Result uncommittedResult1 = prepareResult(ANY_TEXT_1, ANY_TEXT_2, TransactionState.DELETED); + Result uncommittedResult2 = prepareResult(ANY_TEXT_1, ANY_TEXT_3, TransactionState.PREPARED); + Result uncommittedResult3 = prepareResult(ANY_TEXT_1, ANY_TEXT_4, TransactionState.PREPARED); - Result uncommittedResult = prepareResult(ANY_TEXT_1, ANY_TEXT_3, TransactionState.PREPARED); + Snapshot.Key key1 = new Snapshot.Key(scanWithLimit, uncommittedResult1); + Snapshot.Key key2 = new Snapshot.Key(scanWithLimit, uncommittedResult2); + Snapshot.Key key3 = new Snapshot.Key(scanWithLimit, uncommittedResult3); // Set up mock scanner to return one committed and one uncommitted result if (scanType == ScanType.SCAN) { - when(scanner.iterator()).thenReturn(Collections.singletonList(uncommittedResult).iterator()); + when(scanner.iterator()) + .thenReturn( + Arrays.asList(uncommittedResult1, uncommittedResult2, uncommittedResult3).iterator()); } else { - when(scanner.one()).thenReturn(Optional.of(uncommittedResult)).thenReturn(Optional.empty()); + when(scanner.one()) + .thenReturn(Optional.of(uncommittedResult1)) + .thenReturn(Optional.of(uncommittedResult2)) + .thenReturn(Optional.of(uncommittedResult3)) + .thenReturn(Optional.empty()); } - when(storage.scan(scanForStorage)).thenReturn(scanner); + when(storage.scan(scanForStorageWithoutLimit)).thenReturn(scanner); + + when(snapshot.getId()).thenReturn(ANY_ID_1); + + TransactionResult recoveredResult1 = mock(TransactionResult.class); + when(recoveredResult1.getContainedColumnNames()).thenReturn(Collections.singleton(ANY_NAME_3)); + when(recoveredResult1.getAsObject(ANY_NAME_3)).thenReturn(ANY_TEXT_3); + when(recoveredResult1.getColumns()) + .thenReturn(ImmutableMap.of(ANY_NAME_3, TextColumn.of(ANY_NAME_3, ANY_TEXT_3))); - // Act & Assert - assertThatThrownBy(() -> scanOrGetScanner(scanWithLimit, scanType)) - .isInstanceOf(UncommittedRecordException.class); + TransactionResult recoveredResult2 = mock(TransactionResult.class); + when(recoveredResult1.getContainedColumnNames()).thenReturn(Collections.singleton(ANY_NAME_3)); + when(recoveredResult1.getAsObject(ANY_NAME_3)).thenReturn(ANY_TEXT_4); + when(recoveredResult1.getColumns()) + .thenReturn(ImmutableMap.of(ANY_NAME_3, TextColumn.of(ANY_NAME_3, ANY_TEXT_4))); + + @SuppressWarnings("unchecked") + Future recoveryFuture = mock(Future.class); + + when(recoveryExecutor.execute( + key1, scanForStorageWithLimit, new TransactionResult(uncommittedResult1), ANY_ID_1)) + .thenReturn(new RecoveryExecutor.Result(key1, Optional.empty(), recoveryFuture)); + when(recoveryExecutor.execute( + key2, scanForStorageWithLimit, new TransactionResult(uncommittedResult2), ANY_ID_1)) + .thenReturn( + new RecoveryExecutor.Result(key2, Optional.of(recoveredResult1), recoveryFuture)); + when(recoveryExecutor.execute( + key3, scanForStorageWithLimit, new TransactionResult(uncommittedResult3), ANY_ID_1)) + .thenReturn( + new RecoveryExecutor.Result(key3, Optional.of(recoveredResult2), recoveryFuture)); + // Act + List results = scanOrGetScanner(scanWithLimit, scanType); + + // Assert + verify(storage).scan(scanForStorageWithoutLimit); verify(scanner).close(); - verify(snapshot, never()).putIntoReadSet(any(), any()); - verify(snapshot, never()).putIntoScanSet(any(), any()); + verify(snapshot).putIntoReadSet(key2, Optional.of(recoveredResult1)); + verify(snapshot).putIntoReadSet(key3, Optional.of(recoveredResult2)); + verify(snapshot) + .putIntoScanSet( + scanForStorageWithLimit, + Maps.newLinkedHashMap(ImmutableMap.of(key2, recoveredResult1, key3, recoveredResult2))); + verify(snapshot) + .verifyNoOverlap( + scanForStorageWithLimit, + ImmutableMap.of(key2, recoveredResult1, key3, recoveredResult2)); + + assertThat(results) + .containsExactly( + new FilteredResult(recoveredResult1, Collections.emptyList(), TABLE_METADATA, false), + new FilteredResult(recoveredResult2, Collections.emptyList(), TABLE_METADATA, false)); } @Test @@ -1230,6 +1417,7 @@ public void getScanner_ExecutionExceptionThrownByScannerOne_ShouldThrowCrudExcep new CrudHandler( storage, snapshot, + recoveryExecutor, tableMetadataManager, false, mutationConditionsValidator, @@ -1272,6 +1460,7 @@ public void getScanner_ExecutionExceptionThrownByScannerOne_ShouldThrowCrudExcep new CrudHandler( storage, snapshot, + recoveryExecutor, tableMetadataManager, false, mutationConditionsValidator, @@ -1673,8 +1862,8 @@ public void readUnread_GetContainedInGetSet_ShouldCallAppropriateMethods() @Test public void - readUnread_GetNotContainedInGetSet_UncommittedRecordReturnedByStorage_ShouldThrowUncommittedRecordException() - throws ExecutionException { + readUnread_GetNotContainedInGetSet_UncommittedRecordReturnedByStorage_ShouldCallRecoveryExecutor() + throws ExecutionException, CrudException { // Arrange Snapshot.Key key = mock(Snapshot.Key.class); when(key.getNamespace()).thenReturn(ANY_NAMESPACE_NAME); @@ -1691,17 +1880,166 @@ public void readUnread_GetContainedInGetSet_ShouldCallAppropriateMethods() .partitionKey(key.getPartitionKey()) .build(); when(snapshot.containsKeyInGetSet(getForKey)).thenReturn(false); + when(snapshot.getId()).thenReturn(ANY_ID_1); - // Act Assert - assertThatThrownBy(() -> handler.readUnread(key, getForKey)) - .isInstanceOf(UncommittedRecordException.class) - .satisfies( - e -> { - UncommittedRecordException exception = (UncommittedRecordException) e; - assertThat(exception.getSelection()).isEqualTo(getForKey); - assertThat(exception.getResults().size()).isEqualTo(1); - assertThat(exception.getResults().get(0)).isEqualTo(result); - }); + TransactionResult recoveredResult = mock(TransactionResult.class); + @SuppressWarnings("unchecked") + Future recoveryFuture = mock(Future.class); + + when(recoveryExecutor.execute(key, getForKey, new TransactionResult(result), ANY_ID_1)) + .thenReturn(new RecoveryExecutor.Result(key, Optional.of(recoveredResult), recoveryFuture)); + + // Act + handler.readUnread(key, getForKey); + + // Assert + verify(storage).get(getForKey); + verify(recoveryExecutor).execute(key, getForKey, new TransactionResult(result), ANY_ID_1); + verify(snapshot).putIntoReadSet(key, Optional.of(recoveredResult)); + verify(snapshot).putIntoGetSet(getForKey, Optional.of(recoveredResult)); + } + + @Test + public void + readUnread_GetNotContainedInGetSet_UncommittedRecordReturnedByStorage_RecoveredRecordIsEmpty_ShouldCallRecoveryExecutor() + throws ExecutionException, CrudException { + // Arrange + Snapshot.Key key = mock(Snapshot.Key.class); + when(key.getNamespace()).thenReturn(ANY_NAMESPACE_NAME); + when(key.getTable()).thenReturn(ANY_TABLE_NAME); + when(key.getPartitionKey()).thenReturn(Key.ofText(ANY_NAME_1, ANY_TEXT_1)); + + when(result.getInt(Attribute.STATE)).thenReturn(TransactionState.PREPARED.get()); + when(storage.get(any())).thenReturn(Optional.of(result)); + + Get getForKey = + Get.newBuilder() + .namespace(key.getNamespace()) + .table(key.getTable()) + .partitionKey(key.getPartitionKey()) + .build(); + when(snapshot.containsKeyInGetSet(getForKey)).thenReturn(false); + when(snapshot.getId()).thenReturn(ANY_ID_1); + + Optional recoveredRecord = Optional.empty(); + @SuppressWarnings("unchecked") + Future recoveryFuture = mock(Future.class); + + when(recoveryExecutor.execute(key, getForKey, new TransactionResult(result), ANY_ID_1)) + .thenReturn(new RecoveryExecutor.Result(key, recoveredRecord, recoveryFuture)); + + // Act + handler.readUnread(key, getForKey); + + // Assert + verify(storage).get(getForKey); + verify(recoveryExecutor).execute(key, getForKey, new TransactionResult(result), ANY_ID_1); + verify(snapshot).putIntoReadSet(key, recoveredRecord); + verify(snapshot).putIntoGetSet(getForKey, recoveredRecord); + } + + @Test + public void + readUnread_GetWithConjunctionGiven_GetNotContainedInGetSet_UncommittedRecordReturnedByStorage_RecoveredRecordMatchesConjunction_ShouldCallRecoveryExecutor() + throws ExecutionException, CrudException { + // Arrange + Snapshot.Key key = mock(Snapshot.Key.class); + when(key.getNamespace()).thenReturn(ANY_NAMESPACE_NAME); + when(key.getTable()).thenReturn(ANY_TABLE_NAME); + when(key.getPartitionKey()).thenReturn(Key.ofText(ANY_NAME_1, ANY_TEXT_1)); + + when(result.getInt(Attribute.STATE)).thenReturn(TransactionState.PREPARED.get()); + when(storage.get(any())).thenReturn(Optional.of(result)); + + Get getWithConjunction = + Get.newBuilder() + .namespace(key.getNamespace()) + .table(key.getTable()) + .partitionKey(key.getPartitionKey()) + .where(column(ANY_NAME_3).isEqualToText(ANY_TEXT_3)) + .build(); + when(snapshot.containsKeyInGetSet(getWithConjunction)).thenReturn(false); + when(snapshot.getId()).thenReturn(ANY_ID_1); + + TransactionResult recoveredResult = mock(TransactionResult.class); + when(recoveredResult.getContainedColumnNames()).thenReturn(Collections.singleton(ANY_NAME_3)); + when(recoveredResult.getAsObject(ANY_NAME_3)).thenReturn(ANY_TEXT_3); + when(recoveredResult.getColumns()) + .thenReturn(ImmutableMap.of(ANY_NAME_3, TextColumn.of(ANY_NAME_3, ANY_TEXT_3))); + + @SuppressWarnings("unchecked") + Future recoveryFuture = mock(Future.class); + + when(recoveryExecutor.execute(key, getWithConjunction, new TransactionResult(result), ANY_ID_1)) + .thenReturn(new RecoveryExecutor.Result(key, Optional.of(recoveredResult), recoveryFuture)); + + // Act + handler.readUnread(key, getWithConjunction); + + // Assert + verify(storage) + .get( + Get.newBuilder(getWithConjunction) + .clearConditions() + .where(column(ANY_NAME_3).isEqualToText(ANY_TEXT_3)) + .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isEqualToText(ANY_TEXT_3)) + .build()); + verify(recoveryExecutor) + .execute(key, getWithConjunction, new TransactionResult(result), ANY_ID_1); + verify(snapshot).putIntoReadSet(key, Optional.of(recoveredResult)); + verify(snapshot).putIntoGetSet(getWithConjunction, Optional.of(recoveredResult)); + } + + @Test + public void + readUnread_GetWithConjunctionGiven_GetNotContainedInGetSet_UncommittedRecordReturnedByStorage_RecoveredRecordDoesNotMatchConjunction_ShouldCallRecoveryExecutor() + throws ExecutionException, CrudException { + // Arrange + Snapshot.Key key = mock(Snapshot.Key.class); + when(key.getNamespace()).thenReturn(ANY_NAMESPACE_NAME); + when(key.getTable()).thenReturn(ANY_TABLE_NAME); + when(key.getPartitionKey()).thenReturn(Key.ofText(ANY_NAME_1, ANY_TEXT_1)); + + when(result.getInt(Attribute.STATE)).thenReturn(TransactionState.PREPARED.get()); + when(storage.get(any())).thenReturn(Optional.of(result)); + + Get getWithConjunction = + Get.newBuilder() + .namespace(key.getNamespace()) + .table(key.getTable()) + .partitionKey(key.getPartitionKey()) + .where(column(ANY_NAME_3).isEqualToText(ANY_TEXT_3)) + .build(); + when(snapshot.containsKeyInGetSet(getWithConjunction)).thenReturn(false); + when(snapshot.getId()).thenReturn(ANY_ID_1); + + TransactionResult recoveredResult = mock(TransactionResult.class); + when(recoveredResult.getContainedColumnNames()).thenReturn(Collections.singleton(ANY_NAME_3)); + when(recoveredResult.getAsObject(ANY_NAME_3)).thenReturn(ANY_TEXT_4); + when(recoveredResult.getColumns()) + .thenReturn(ImmutableMap.of(ANY_NAME_3, TextColumn.of(ANY_NAME_3, ANY_TEXT_4))); + + @SuppressWarnings("unchecked") + Future recoveryFuture = mock(Future.class); + + when(recoveryExecutor.execute(key, getWithConjunction, new TransactionResult(result), ANY_ID_1)) + .thenReturn(new RecoveryExecutor.Result(key, Optional.of(recoveredResult), recoveryFuture)); + + // Act + handler.readUnread(key, getWithConjunction); + + // Assert + verify(storage) + .get( + Get.newBuilder(getWithConjunction) + .clearConditions() + .where(column(ANY_NAME_3).isEqualToText(ANY_TEXT_3)) + .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isEqualToText(ANY_TEXT_3)) + .build()); + verify(recoveryExecutor) + .execute(key, getWithConjunction, new TransactionResult(result), ANY_ID_1); + verify(snapshot, never()).putIntoReadSet(any(), any()); + verify(snapshot).putIntoGetSet(getWithConjunction, Optional.empty()); } @Test @@ -1758,10 +2096,12 @@ public void readUnread_GetContainedInGetSet_ShouldCallAppropriateMethods() @Test public void - readUnread_NullKeyAndGetWithIndexNotContainedInGetSet_UncommittedRecordReturnedByStorage_ShouldThrowUncommittedRecordException() - throws ExecutionException { + readUnread_NullKeyAndGetWithIndexNotContainedInGetSet_UncommittedRecordReturnedByStorage_ShouldCallRecoveryExecutor() + throws ExecutionException, CrudException { // Arrange when(result.getInt(Attribute.STATE)).thenReturn(TransactionState.PREPARED.get()); + when(result.getPartitionKey()).thenReturn(Optional.of(Key.ofText(ANY_NAME_1, ANY_TEXT_1))); + when(result.getClusteringKey()).thenReturn(Optional.of(Key.ofText(ANY_NAME_2, ANY_TEXT_2))); when(storage.get(any())).thenReturn(Optional.of(result)); Get getWithIndex = @@ -1771,17 +2111,25 @@ public void readUnread_GetContainedInGetSet_ShouldCallAppropriateMethods() .indexKey(Key.ofText(ANY_NAME_3, ANY_TEXT_1)) .build(); when(snapshot.containsKeyInGetSet(getWithIndex)).thenReturn(false); + when(snapshot.getId()).thenReturn(ANY_ID_1); - // Act Assert - assertThatThrownBy(() -> handler.readUnread(null, getWithIndex)) - .isInstanceOf(UncommittedRecordException.class) - .satisfies( - e -> { - UncommittedRecordException exception = (UncommittedRecordException) e; - assertThat(exception.getSelection()).isEqualTo(getWithIndex); - assertThat(exception.getResults().size()).isEqualTo(1); - assertThat(exception.getResults().get(0)).isEqualTo(result); - }); + Snapshot.Key key = new Snapshot.Key(getWithIndex, result); + + TransactionResult recoveredResult = mock(TransactionResult.class); + @SuppressWarnings("unchecked") + Future recoveryFuture = mock(Future.class); + + when(recoveryExecutor.execute(key, getWithIndex, new TransactionResult(result), ANY_ID_1)) + .thenReturn(new RecoveryExecutor.Result(key, Optional.of(recoveredResult), recoveryFuture)); + + // Act + handler.readUnread(key, getWithIndex); + + // Assert + verify(storage).get(getWithIndex); + verify(recoveryExecutor).execute(key, getWithIndex, new TransactionResult(result), ANY_ID_1); + verify(snapshot).putIntoReadSet(key, Optional.of(recoveredResult)); + verify(snapshot).putIntoGetSet(getWithIndex, Optional.of(recoveredResult)); } @Test @@ -2010,7 +2358,6 @@ public void get_WithConjunctions_ShouldConvertConjunctions() .clusteringKey(Key.ofText(ANY_NAME_2, ANY_TEXT_2)) .where(column(ANY_NAME_3).isEqualToText(ANY_TEXT_3)) .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isEqualToText(ANY_TEXT_3)) - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .consistency(Consistency.LINEARIZABLE) .build()); verify(storage) @@ -2029,7 +2376,6 @@ public void get_WithConjunctions_ShouldConvertConjunctions() column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isEqualToText(ANY_TEXT_3)) .and(column(Attribute.BEFORE_PREFIX + ANY_NAME_4).isEqualToInt(10)) .build()) - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .consistency(Consistency.LINEARIZABLE) .build()); verify(storage) @@ -2043,7 +2389,6 @@ public void get_WithConjunctions_ShouldConvertConjunctions() .or(column(ANY_NAME_4).isEqualToInt(20)) .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isEqualToText(ANY_TEXT_3)) .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_4).isEqualToInt(20)) - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .consistency(Consistency.LINEARIZABLE) .build()); verify(storage) @@ -2074,7 +2419,6 @@ public void get_WithConjunctions_ShouldConvertConjunctions() .and( column(Attribute.BEFORE_PREFIX + ANY_NAME_4).isLessThanOrEqualToInt(40)) .build()) - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .consistency(Consistency.LINEARIZABLE) .build()); verify(storage) @@ -2122,7 +2466,6 @@ public void get_WithConjunctions_ShouldConvertConjunctions() column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isEqualToText(ANY_TEXT_4)) .and(column(Attribute.BEFORE_PREFIX + ANY_NAME_4).isGreaterThanInt(60)) .build()) - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .consistency(Consistency.LINEARIZABLE) .build()); verify(storage) @@ -2136,7 +2479,6 @@ public void get_WithConjunctions_ShouldConvertConjunctions() .or(column(ANY_NAME_3).isLikeText(ANY_TEXT_4)) .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isLikeText(ANY_TEXT_3)) .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isLikeText(ANY_TEXT_4)) - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .consistency(Consistency.LINEARIZABLE) .build()); } @@ -2237,7 +2579,6 @@ public void scan_WithConjunctions_ShouldConvertConjunctions() .partitionKey(Key.ofText(ANY_NAME_1, ANY_TEXT_1)) .where(column(ANY_NAME_3).isEqualToText(ANY_TEXT_3)) .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isEqualToText(ANY_TEXT_3)) - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .consistency(Consistency.LINEARIZABLE) .build()); verify(storage) @@ -2255,7 +2596,6 @@ public void scan_WithConjunctions_ShouldConvertConjunctions() column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isEqualToText(ANY_TEXT_3)) .and(column(Attribute.BEFORE_PREFIX + ANY_NAME_4).isEqualToInt(10)) .build()) - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .consistency(Consistency.LINEARIZABLE) .build()); verify(storage) @@ -2268,7 +2608,6 @@ public void scan_WithConjunctions_ShouldConvertConjunctions() .or(column(ANY_NAME_4).isEqualToInt(20)) .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isEqualToText(ANY_TEXT_3)) .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_4).isEqualToInt(20)) - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .consistency(Consistency.LINEARIZABLE) .build()); verify(storage) @@ -2298,7 +2637,6 @@ public void scan_WithConjunctions_ShouldConvertConjunctions() .and( column(Attribute.BEFORE_PREFIX + ANY_NAME_4).isLessThanOrEqualToInt(40)) .build()) - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .consistency(Consistency.LINEARIZABLE) .build()); verify(storage) @@ -2345,7 +2683,6 @@ public void scan_WithConjunctions_ShouldConvertConjunctions() column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isEqualToText(ANY_TEXT_4)) .and(column(Attribute.BEFORE_PREFIX + ANY_NAME_4).isGreaterThanInt(60)) .build()) - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .consistency(Consistency.LINEARIZABLE) .build()); verify(storage) @@ -2358,7 +2695,6 @@ public void scan_WithConjunctions_ShouldConvertConjunctions() .or(column(ANY_NAME_3).isLikeText(ANY_TEXT_4)) .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isLikeText(ANY_TEXT_3)) .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isLikeText(ANY_TEXT_4)) - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .consistency(Consistency.LINEARIZABLE) .build()); verify(storage) @@ -2369,7 +2705,6 @@ public void scan_WithConjunctions_ShouldConvertConjunctions() .all() .where(column(ANY_NAME_1).isGreaterThanText(ANY_TEXT_3)) .and(column(ANY_NAME_2).isLessThanOrEqualToText(ANY_TEXT_4)) - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .consistency(Consistency.LINEARIZABLE) .build()); verify(storage) @@ -2386,7 +2721,6 @@ public void scan_WithConjunctions_ShouldConvertConjunctions() condition(column(ANY_NAME_1).isGreaterThanText(ANY_TEXT_3)) .and(column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isEqualToText(ANY_TEXT_4)) .build()) - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .consistency(Consistency.LINEARIZABLE) .build()); } diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutorTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutorTest.java new file mode 100644 index 0000000000..45ec9aaff4 --- /dev/null +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutorTest.java @@ -0,0 +1,481 @@ +package com.scalar.db.transaction.consensuscommit; + +import static com.scalar.db.transaction.consensuscommit.Attribute.BEFORE_COMMITTED_AT; +import static com.scalar.db.transaction.consensuscommit.Attribute.BEFORE_ID; +import static com.scalar.db.transaction.consensuscommit.Attribute.BEFORE_PREFIX; +import static com.scalar.db.transaction.consensuscommit.Attribute.BEFORE_PREPARED_AT; +import static com.scalar.db.transaction.consensuscommit.Attribute.BEFORE_STATE; +import static com.scalar.db.transaction.consensuscommit.Attribute.BEFORE_VERSION; +import static com.scalar.db.transaction.consensuscommit.Attribute.COMMITTED_AT; +import static com.scalar.db.transaction.consensuscommit.Attribute.ID; +import static com.scalar.db.transaction.consensuscommit.Attribute.PREPARED_AT; +import static com.scalar.db.transaction.consensuscommit.Attribute.STATE; +import static com.scalar.db.transaction.consensuscommit.Attribute.VERSION; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import com.scalar.db.api.Selection; +import com.scalar.db.api.TableMetadata; +import com.scalar.db.api.TransactionState; +import com.scalar.db.common.ResultImpl; +import com.scalar.db.exception.storage.ExecutionException; +import com.scalar.db.exception.transaction.CrudException; +import com.scalar.db.io.BigIntColumn; +import com.scalar.db.io.BlobColumn; +import com.scalar.db.io.BooleanColumn; +import com.scalar.db.io.Column; +import com.scalar.db.io.DataType; +import com.scalar.db.io.DateColumn; +import com.scalar.db.io.DoubleColumn; +import com.scalar.db.io.FloatColumn; +import com.scalar.db.io.IntColumn; +import com.scalar.db.io.TextColumn; +import com.scalar.db.io.TimeColumn; +import com.scalar.db.io.TimestampColumn; +import com.scalar.db.io.TimestampTZColumn; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +public class RecoveryExecutorTest { + + private static final String ANY_NAMESPACE_NAME = "namespace"; + private static final String ANY_TABLE_NAME = "table"; + private static final String ANY_ID_1 = "id1"; + private static final String ANY_ID_2 = "id2"; + private static final String ANY_ID_3 = "id3"; + private static final long ANY_TIME_MILLIS_1 = 100; + private static final long ANY_TIME_MILLIS_2 = 200; + private static final long ANY_TIME_MILLIS_3 = 300; + private static final long ANY_TIME_MILLIS_4 = 400; + private static final String ANY_NAME_1 = "name1"; + private static final String ANY_NAME_2 = "name2"; + private static final String ANY_NAME_3 = "name3"; + private static final String ANY_NAME_4 = "name4"; + private static final String ANY_NAME_5 = "name5"; + private static final String ANY_NAME_6 = "name6"; + private static final String ANY_NAME_7 = "name7"; + private static final String ANY_NAME_8 = "name8"; + private static final String ANY_NAME_9 = "name9"; + private static final String ANY_NAME_10 = "name10"; + private static final String ANY_NAME_11 = "name11"; + private static final String ANY_NAME_12 = "name12"; + private static final String ANY_NAME_13 = "name13"; + private static final String ANY_TEXT_1 = "text1"; + private static final String ANY_TEXT_2 = "text2"; + private static final String ANY_TEXT_3 = "text3"; + private static final String ANY_TEXT_4 = "text4"; + private static final int ANY_INT_1 = 100; + private static final int ANY_INT_2 = 200; + private static final long ANY_BIGINT_1 = 1000L; + private static final long ANY_BIGINT_2 = 2000L; + private static final float ANY_FLOAT_1 = 1.23f; + private static final float ANY_FLOAT_2 = 4.56f; + private static final double ANY_DOUBLE_1 = 7.89; + private static final double ANY_DOUBLE_2 = 0.12; + private static final byte[] ANY_BLOB_1 = new byte[] {1, 2, 3}; + private static final byte[] ANY_BLOB_2 = new byte[] {4, 5, 6}; + private static final LocalDate ANY_DATE_1 = LocalDate.of(2020, 1, 1); + private static final LocalDate ANY_DATE_2 = LocalDate.of(2021, 1, 1); + private static final LocalTime ANY_TIME_1 = LocalTime.of(12, 0, 0); + private static final LocalTime ANY_TIME_2 = LocalTime.of(13, 0, 0); + private static final LocalDateTime ANY_TIMESTAMP_1 = LocalDateTime.of(2020, 1, 1, 12, 0, 0); + private static final LocalDateTime ANY_TIMESTAMP_2 = LocalDateTime.of(2021, 1, 1, 13, 0, 0); + private static final Instant ANY_TIMESTAMPTZ_1 = + LocalDateTime.of(2020, 1, 1, 12, 0, 0).toInstant(ZoneOffset.UTC); + private static final Instant ANY_TIMESTAMPTZ_2 = + LocalDateTime.of(2021, 1, 1, 13, 0, 0).toInstant(ZoneOffset.UTC); + + private static final TableMetadata TABLE_METADATA = + ConsensusCommitUtils.buildTransactionTableMetadata( + TableMetadata.newBuilder() + .addColumn(ANY_NAME_1, DataType.TEXT) + .addColumn(ANY_NAME_2, DataType.TEXT) + .addColumn(ANY_NAME_3, DataType.INT) + .addColumn(ANY_NAME_4, DataType.BOOLEAN) + .addColumn(ANY_NAME_5, DataType.BIGINT) + .addColumn(ANY_NAME_6, DataType.FLOAT) + .addColumn(ANY_NAME_7, DataType.DOUBLE) + .addColumn(ANY_NAME_8, DataType.TEXT) + .addColumn(ANY_NAME_9, DataType.BLOB) + .addColumn(ANY_NAME_10, DataType.DATE) + .addColumn(ANY_NAME_11, DataType.TIME) + .addColumn(ANY_NAME_12, DataType.TIMESTAMP) + .addColumn(ANY_NAME_13, DataType.TIMESTAMPTZ) + .addPartitionKey(ANY_NAME_1) + .addClusteringKey(ANY_NAME_2) + .build()); + + @Mock private Coordinator coordinator; + @Mock private RecoveryHandler recovery; + @Mock private TransactionTableMetadataManager tableMetadataManager; + @Mock private Snapshot.Key snapshotKey; + @Mock private Selection selection; + + private RecoveryExecutor executor; + + @BeforeEach + public void setUp() throws Exception { + MockitoAnnotations.openMocks(this).close(); + + executor = new RecoveryExecutor(coordinator, recovery, tableMetadataManager, 1); + + // Arrange + when(tableMetadataManager.getTransactionTableMetadata(selection)) + .thenReturn(new TransactionTableMetadata(TABLE_METADATA)); + + when(selection.forNamespace()).thenReturn(Optional.of(ANY_NAMESPACE_NAME)); + when(selection.forTable()).thenReturn(Optional.of(ANY_TABLE_NAME)); + when(selection.forFullTableName()) + .thenReturn(Optional.of(ANY_NAMESPACE_NAME + "." + ANY_TABLE_NAME)); + } + + private TransactionResult prepareResult(TransactionState state) { + ImmutableMap> columns = + ImmutableMap.>builder() + .put(ANY_NAME_1, TextColumn.of(ANY_NAME_1, ANY_TEXT_1)) + .put(ANY_NAME_2, TextColumn.of(ANY_NAME_2, ANY_TEXT_2)) + .put(ANY_NAME_3, IntColumn.of(ANY_NAME_3, ANY_INT_2)) + .put(ANY_NAME_4, BooleanColumn.of(ANY_NAME_4, true)) + .put(ANY_NAME_5, BigIntColumn.of(ANY_NAME_5, ANY_BIGINT_2)) + .put(ANY_NAME_6, FloatColumn.of(ANY_NAME_6, ANY_FLOAT_2)) + .put(ANY_NAME_7, DoubleColumn.of(ANY_NAME_7, ANY_DOUBLE_2)) + .put(ANY_NAME_8, TextColumn.of(ANY_NAME_8, ANY_TEXT_4)) + .put(ANY_NAME_9, BlobColumn.of(ANY_NAME_9, ANY_BLOB_2)) + .put(ANY_NAME_10, DateColumn.of(ANY_NAME_10, ANY_DATE_2)) + .put(ANY_NAME_11, TimeColumn.of(ANY_NAME_11, ANY_TIME_2)) + .put(ANY_NAME_12, TimestampColumn.of(ANY_NAME_12, ANY_TIMESTAMP_2)) + .put(ANY_NAME_13, TimestampTZColumn.of(ANY_NAME_13, ANY_TIMESTAMPTZ_2)) + .put(ID, TextColumn.of(ID, ANY_ID_2)) + .put(PREPARED_AT, BigIntColumn.of(PREPARED_AT, ANY_TIME_MILLIS_3)) + .put(STATE, IntColumn.of(STATE, state.get())) + .put(VERSION, IntColumn.of(VERSION, 1)) + .put(BEFORE_PREFIX + ANY_NAME_3, IntColumn.of(BEFORE_PREFIX + ANY_NAME_3, ANY_INT_1)) + .put(BEFORE_PREFIX + ANY_NAME_4, BooleanColumn.of(BEFORE_PREFIX + ANY_NAME_4, false)) + .put( + BEFORE_PREFIX + ANY_NAME_5, + BigIntColumn.of(BEFORE_PREFIX + ANY_NAME_5, ANY_BIGINT_1)) + .put( + BEFORE_PREFIX + ANY_NAME_6, FloatColumn.of(BEFORE_PREFIX + ANY_NAME_6, ANY_FLOAT_1)) + .put( + BEFORE_PREFIX + ANY_NAME_7, + DoubleColumn.of(BEFORE_PREFIX + ANY_NAME_7, ANY_DOUBLE_1)) + .put(BEFORE_PREFIX + ANY_NAME_8, TextColumn.of(BEFORE_PREFIX + ANY_NAME_8, ANY_TEXT_3)) + .put(BEFORE_PREFIX + ANY_NAME_9, BlobColumn.of(BEFORE_PREFIX + ANY_NAME_9, ANY_BLOB_1)) + .put( + BEFORE_PREFIX + ANY_NAME_10, DateColumn.of(BEFORE_PREFIX + ANY_NAME_10, ANY_DATE_1)) + .put( + BEFORE_PREFIX + ANY_NAME_11, TimeColumn.of(BEFORE_PREFIX + ANY_NAME_11, ANY_TIME_1)) + .put( + BEFORE_PREFIX + ANY_NAME_12, + TimestampColumn.of(BEFORE_PREFIX + ANY_NAME_12, ANY_TIMESTAMP_1)) + .put( + BEFORE_PREFIX + ANY_NAME_13, + TimestampTZColumn.of(BEFORE_PREFIX + ANY_NAME_13, ANY_TIMESTAMPTZ_1)) + .put(BEFORE_ID, TextColumn.of(BEFORE_ID, ANY_ID_1)) + .put(BEFORE_PREPARED_AT, BigIntColumn.of(BEFORE_PREPARED_AT, ANY_TIME_MILLIS_1)) + .put(BEFORE_COMMITTED_AT, BigIntColumn.of(BEFORE_COMMITTED_AT, ANY_TIME_MILLIS_2)) + .put(BEFORE_STATE, IntColumn.of(BEFORE_STATE, TransactionState.COMMITTED.get())) + .put(BEFORE_VERSION, IntColumn.of(BEFORE_VERSION, 1)) + .build(); + return new TransactionResult(new ResultImpl(columns, TABLE_METADATA)); + } + + private TransactionResult prepareResultWithoutBeforeImage(TransactionState state) { + ImmutableMap> columns = + ImmutableMap.>builder() + .put(ANY_NAME_1, TextColumn.of(ANY_NAME_1, ANY_TEXT_1)) + .put(ANY_NAME_2, TextColumn.of(ANY_NAME_2, ANY_TEXT_2)) + .put(ANY_NAME_3, IntColumn.of(ANY_NAME_3, ANY_INT_2)) + .put(ANY_NAME_4, BooleanColumn.of(ANY_NAME_4, true)) + .put(ANY_NAME_5, BigIntColumn.of(ANY_NAME_5, ANY_BIGINT_2)) + .put(ANY_NAME_6, FloatColumn.of(ANY_NAME_6, ANY_FLOAT_2)) + .put(ANY_NAME_7, DoubleColumn.of(ANY_NAME_7, ANY_DOUBLE_2)) + .put(ANY_NAME_8, TextColumn.of(ANY_NAME_8, ANY_TEXT_4)) + .put(ANY_NAME_9, BlobColumn.of(ANY_NAME_9, ANY_BLOB_2)) + .put(ANY_NAME_10, DateColumn.of(ANY_NAME_10, ANY_DATE_2)) + .put(ANY_NAME_11, TimeColumn.of(ANY_NAME_11, ANY_TIME_2)) + .put(ANY_NAME_12, TimestampColumn.of(ANY_NAME_12, ANY_TIMESTAMP_2)) + .put(ANY_NAME_13, TimestampTZColumn.of(ANY_NAME_13, ANY_TIMESTAMPTZ_2)) + .put(ID, TextColumn.of(ID, ANY_ID_2)) + .put(PREPARED_AT, BigIntColumn.of(PREPARED_AT, ANY_TIME_MILLIS_3)) + .put(STATE, IntColumn.of(STATE, state.get())) + .put(VERSION, IntColumn.of(VERSION, 1)) + .put(BEFORE_PREFIX + ANY_NAME_3, IntColumn.ofNull(BEFORE_PREFIX + ANY_NAME_3)) + .put(BEFORE_PREFIX + ANY_NAME_4, BooleanColumn.ofNull(BEFORE_PREFIX + ANY_NAME_4)) + .put(BEFORE_PREFIX + ANY_NAME_5, BigIntColumn.ofNull(BEFORE_PREFIX + ANY_NAME_5)) + .put(BEFORE_PREFIX + ANY_NAME_6, FloatColumn.ofNull(BEFORE_PREFIX + ANY_NAME_6)) + .put(BEFORE_PREFIX + ANY_NAME_7, DoubleColumn.ofNull(BEFORE_PREFIX + ANY_NAME_7)) + .put(BEFORE_PREFIX + ANY_NAME_8, TextColumn.ofNull(BEFORE_PREFIX + ANY_NAME_8)) + .put(BEFORE_PREFIX + ANY_NAME_9, BlobColumn.ofNull(BEFORE_PREFIX + ANY_NAME_9)) + .put(BEFORE_PREFIX + ANY_NAME_10, DateColumn.ofNull(BEFORE_PREFIX + ANY_NAME_10)) + .put(BEFORE_PREFIX + ANY_NAME_11, TimeColumn.ofNull(BEFORE_PREFIX + ANY_NAME_11)) + .put(BEFORE_PREFIX + ANY_NAME_12, TimestampColumn.ofNull(BEFORE_PREFIX + ANY_NAME_12)) + .put(BEFORE_PREFIX + ANY_NAME_13, TimestampTZColumn.ofNull(BEFORE_PREFIX + ANY_NAME_13)) + .put(BEFORE_ID, TextColumn.ofNull(BEFORE_ID)) + .put(BEFORE_PREPARED_AT, BigIntColumn.ofNull(BEFORE_PREPARED_AT)) + .put(BEFORE_COMMITTED_AT, BigIntColumn.ofNull(BEFORE_COMMITTED_AT)) + .put(BEFORE_STATE, IntColumn.ofNull(BEFORE_STATE)) + .put(BEFORE_VERSION, IntColumn.ofNull(BEFORE_VERSION)) + .build(); + return new TransactionResult(new ResultImpl(columns, TABLE_METADATA)); + } + + private TransactionResult prepareRolledBackResult() { + ImmutableMap> columns = + ImmutableMap.>builder() + .put(ANY_NAME_1, TextColumn.of(ANY_NAME_1, ANY_TEXT_1)) + .put(ANY_NAME_2, TextColumn.of(ANY_NAME_2, ANY_TEXT_2)) + .put(ANY_NAME_3, IntColumn.of(ANY_NAME_3, ANY_INT_1)) + .put(ANY_NAME_4, BooleanColumn.of(ANY_NAME_4, false)) + .put(ANY_NAME_5, BigIntColumn.of(ANY_NAME_5, ANY_BIGINT_1)) + .put(ANY_NAME_6, FloatColumn.of(ANY_NAME_6, ANY_FLOAT_1)) + .put(ANY_NAME_7, DoubleColumn.of(ANY_NAME_7, ANY_DOUBLE_1)) + .put(ANY_NAME_8, TextColumn.of(ANY_NAME_8, ANY_TEXT_3)) + .put(ANY_NAME_9, BlobColumn.of(ANY_NAME_9, ANY_BLOB_1)) + .put(ANY_NAME_10, DateColumn.of(ANY_NAME_10, ANY_DATE_1)) + .put(ANY_NAME_11, TimeColumn.of(ANY_NAME_11, ANY_TIME_1)) + .put(ANY_NAME_12, TimestampColumn.of(ANY_NAME_12, ANY_TIMESTAMP_1)) + .put(ANY_NAME_13, TimestampTZColumn.of(ANY_NAME_13, ANY_TIMESTAMPTZ_1)) + .put(ID, TextColumn.of(ID, ANY_ID_1)) + .put(PREPARED_AT, BigIntColumn.of(PREPARED_AT, ANY_TIME_MILLIS_1)) + .put(COMMITTED_AT, BigIntColumn.of(COMMITTED_AT, ANY_TIME_MILLIS_2)) + .put(STATE, IntColumn.of(STATE, TransactionState.COMMITTED.get())) + .put(VERSION, IntColumn.of(VERSION, 1)) + .put(BEFORE_PREFIX + ANY_NAME_3, IntColumn.ofNull(BEFORE_PREFIX + ANY_NAME_3)) + .put(BEFORE_PREFIX + ANY_NAME_4, BooleanColumn.ofNull(BEFORE_PREFIX + ANY_NAME_4)) + .put(BEFORE_PREFIX + ANY_NAME_5, BigIntColumn.ofNull(BEFORE_PREFIX + ANY_NAME_5)) + .put(BEFORE_PREFIX + ANY_NAME_6, FloatColumn.ofNull(BEFORE_PREFIX + ANY_NAME_6)) + .put(BEFORE_PREFIX + ANY_NAME_7, DoubleColumn.ofNull(BEFORE_PREFIX + ANY_NAME_7)) + .put(BEFORE_PREFIX + ANY_NAME_8, TextColumn.ofNull(BEFORE_PREFIX + ANY_NAME_8)) + .put(BEFORE_PREFIX + ANY_NAME_9, BlobColumn.ofNull(BEFORE_PREFIX + ANY_NAME_9)) + .put(BEFORE_PREFIX + ANY_NAME_10, DateColumn.ofNull(BEFORE_PREFIX + ANY_NAME_10)) + .put(BEFORE_PREFIX + ANY_NAME_11, TimeColumn.ofNull(BEFORE_PREFIX + ANY_NAME_11)) + .put(BEFORE_PREFIX + ANY_NAME_12, TimestampColumn.ofNull(BEFORE_PREFIX + ANY_NAME_12)) + .put(BEFORE_PREFIX + ANY_NAME_13, TimestampTZColumn.ofNull(BEFORE_PREFIX + ANY_NAME_13)) + .put(BEFORE_ID, TextColumn.ofNull(BEFORE_ID)) + .put(BEFORE_PREPARED_AT, BigIntColumn.ofNull(BEFORE_PREPARED_AT)) + .put(BEFORE_COMMITTED_AT, BigIntColumn.ofNull(BEFORE_COMMITTED_AT)) + .put(BEFORE_STATE, IntColumn.ofNull(BEFORE_STATE)) + .put(BEFORE_VERSION, IntColumn.ofNull(BEFORE_VERSION)) + .build(); + return new TransactionResult(new ResultImpl(columns, TABLE_METADATA)); + } + + private TransactionResult prepareRolledForwardResult() { + ImmutableMap> columns = + ImmutableMap.>builder() + .put(ANY_NAME_1, TextColumn.of(ANY_NAME_1, ANY_TEXT_1)) + .put(ANY_NAME_2, TextColumn.of(ANY_NAME_2, ANY_TEXT_2)) + .put(ANY_NAME_3, IntColumn.of(ANY_NAME_3, ANY_INT_2)) + .put(ANY_NAME_4, BooleanColumn.of(ANY_NAME_4, true)) + .put(ANY_NAME_5, BigIntColumn.of(ANY_NAME_5, ANY_BIGINT_2)) + .put(ANY_NAME_6, FloatColumn.of(ANY_NAME_6, ANY_FLOAT_2)) + .put(ANY_NAME_7, DoubleColumn.of(ANY_NAME_7, ANY_DOUBLE_2)) + .put(ANY_NAME_8, TextColumn.of(ANY_NAME_8, ANY_TEXT_4)) + .put(ANY_NAME_9, BlobColumn.of(ANY_NAME_9, ANY_BLOB_2)) + .put(ANY_NAME_10, DateColumn.of(ANY_NAME_10, ANY_DATE_2)) + .put(ANY_NAME_11, TimeColumn.of(ANY_NAME_11, ANY_TIME_2)) + .put(ANY_NAME_12, TimestampColumn.of(ANY_NAME_12, ANY_TIMESTAMP_2)) + .put(ANY_NAME_13, TimestampTZColumn.of(ANY_NAME_13, ANY_TIMESTAMPTZ_2)) + .put(ID, TextColumn.of(ID, ANY_ID_2)) + .put(PREPARED_AT, BigIntColumn.of(PREPARED_AT, ANY_TIME_MILLIS_3)) + .put(COMMITTED_AT, BigIntColumn.of(COMMITTED_AT, ANY_TIME_MILLIS_4)) + .put(STATE, IntColumn.of(STATE, TransactionState.COMMITTED.get())) + .put(VERSION, IntColumn.of(VERSION, 1)) + .put(BEFORE_PREFIX + ANY_NAME_3, IntColumn.ofNull(BEFORE_PREFIX + ANY_NAME_3)) + .put(BEFORE_PREFIX + ANY_NAME_4, BooleanColumn.ofNull(BEFORE_PREFIX + ANY_NAME_4)) + .put(BEFORE_PREFIX + ANY_NAME_5, BigIntColumn.ofNull(BEFORE_PREFIX + ANY_NAME_5)) + .put(BEFORE_PREFIX + ANY_NAME_6, FloatColumn.ofNull(BEFORE_PREFIX + ANY_NAME_6)) + .put(BEFORE_PREFIX + ANY_NAME_7, DoubleColumn.ofNull(BEFORE_PREFIX + ANY_NAME_7)) + .put(BEFORE_PREFIX + ANY_NAME_8, TextColumn.ofNull(BEFORE_PREFIX + ANY_NAME_8)) + .put(BEFORE_PREFIX + ANY_NAME_9, BlobColumn.ofNull(BEFORE_PREFIX + ANY_NAME_9)) + .put(BEFORE_PREFIX + ANY_NAME_10, DateColumn.ofNull(BEFORE_PREFIX + ANY_NAME_10)) + .put(BEFORE_PREFIX + ANY_NAME_11, TimeColumn.ofNull(BEFORE_PREFIX + ANY_NAME_11)) + .put(BEFORE_PREFIX + ANY_NAME_12, TimestampColumn.ofNull(BEFORE_PREFIX + ANY_NAME_12)) + .put(BEFORE_PREFIX + ANY_NAME_13, TimestampTZColumn.ofNull(BEFORE_PREFIX + ANY_NAME_13)) + .put(BEFORE_ID, TextColumn.ofNull(BEFORE_ID)) + .put(BEFORE_PREPARED_AT, BigIntColumn.ofNull(BEFORE_PREPARED_AT)) + .put(BEFORE_COMMITTED_AT, BigIntColumn.ofNull(BEFORE_COMMITTED_AT)) + .put(BEFORE_STATE, IntColumn.ofNull(BEFORE_STATE)) + .put(BEFORE_VERSION, IntColumn.ofNull(BEFORE_VERSION)) + .build(); + return new TransactionResult(new ResultImpl(columns, TABLE_METADATA)); + } + + @Test + public void execute_CoordinatorExceptionByCoordinatorState_ShouldThrowCrudException() + throws CoordinatorException, ExecutionException { + // Arrange + TransactionResult transactionResult = mock(TransactionResult.class); + when(transactionResult.getId()).thenReturn(ANY_ID_1); + when(coordinator.getState(ANY_ID_1)).thenThrow(new CoordinatorException("error")); + + // Act Assert + assertThatThrownBy(() -> executor.execute(snapshotKey, selection, transactionResult, ANY_ID_3)) + .isInstanceOf(CrudException.class); + + // Verify no recovery attempted + verify(recovery, never()).recover(any(), any(), any()); + } + + @Test + public void + execute_TransactionNotExpiredAndNoCoordinatorState_ShouldThrowUncommittedRecordException() + throws CoordinatorException, ExecutionException { + // Arrange + TransactionResult transactionResult = mock(TransactionResult.class); + when(transactionResult.getId()).thenReturn(ANY_ID_1); + when(coordinator.getState(ANY_ID_1)).thenReturn(Optional.empty()); + when(recovery.isTransactionExpired(transactionResult)).thenReturn(false); + + // Act Assert + assertThatThrownBy(() -> executor.execute(snapshotKey, selection, transactionResult, ANY_ID_3)) + .isInstanceOf(UncommittedRecordException.class); + + // Verify no recovery attempted + verify(recovery, never()).recover(any(), any(), any()); + } + + @Test + public void execute_TransactionExpiredAndNoCoordinatorState_ShouldRollback() throws Exception { + // Arrange + TransactionResult transactionResult = prepareResult(TransactionState.PREPARED); + when(coordinator.getState(ANY_ID_2)).thenReturn(Optional.empty()); + when(recovery.isTransactionExpired(transactionResult)).thenReturn(true); + + // Act + RecoveryExecutor.Result result = + executor.execute(snapshotKey, selection, transactionResult, ANY_ID_3); + + // Wait for recovery to complete + result.recoveryFuture.get(); + + // Assert + assertThat(result.recoveredResult).hasValue(prepareRolledBackResult()); + verify(recovery).recover(eq(selection), eq(transactionResult), eq(Optional.empty())); + } + + @Test + public void + execute_TransactionExpiredAndNoCoordinatorState_RecordWithoutBeforeImage_ShouldRollback() + throws Exception { + // Arrange + TransactionResult transactionResult = + prepareResultWithoutBeforeImage(TransactionState.PREPARED); + when(coordinator.getState(ANY_ID_2)).thenReturn(Optional.empty()); + when(recovery.isTransactionExpired(transactionResult)).thenReturn(true); + + // Act + RecoveryExecutor.Result result = + executor.execute(snapshotKey, selection, transactionResult, ANY_ID_3); + + // Wait for recovery to complete + result.recoveryFuture.get(); + + // Assert + assertThat(result.recoveredResult).isEmpty(); + verify(recovery).recover(eq(selection), eq(transactionResult), eq(Optional.empty())); + } + + @Test + public void execute_CoordinatorStateIsAborted_ShouldRollback() throws Exception { + // Arrange + TransactionResult transactionResult = prepareResult(TransactionState.PREPARED); + Coordinator.State abortedState = new Coordinator.State(ANY_ID_2, TransactionState.ABORTED); + when(coordinator.getState(ANY_ID_2)).thenReturn(Optional.of(abortedState)); + + // Act + RecoveryExecutor.Result result = + executor.execute(snapshotKey, selection, transactionResult, ANY_ID_3); + + // Wait for recovery to complete + result.recoveryFuture.get(); + + // Assert + assertThat(result.recoveredResult).hasValue(prepareRolledBackResult()); + verify(recovery).recover(eq(selection), eq(transactionResult), eq(Optional.of(abortedState))); + } + + @Test + public void execute_CoordinatorStateIsAborted_RecordWithoutBeforeImage_ShouldRollback() + throws Exception { + // Arrange + TransactionResult transactionResult = + prepareResultWithoutBeforeImage(TransactionState.PREPARED); + Coordinator.State abortedState = new Coordinator.State(ANY_ID_2, TransactionState.ABORTED); + when(coordinator.getState(ANY_ID_2)).thenReturn(Optional.of(abortedState)); + + // Act + RecoveryExecutor.Result result = + executor.execute(snapshotKey, selection, transactionResult, ANY_ID_3); + + // Wait for recovery to complete + result.recoveryFuture.get(); + + // Assert + assertThat(result.recoveredResult).isEmpty(); + verify(recovery).recover(eq(selection), eq(transactionResult), eq(Optional.of(abortedState))); + } + + @Test + public void execute_CoordinatorStateIsCommitted_RecordWithPreparedState_ShouldCommit() + throws Exception { + // Arrange + TransactionResult transactionResult = prepareResult(TransactionState.PREPARED); + Coordinator.State commitState = new Coordinator.State(ANY_ID_2, TransactionState.COMMITTED); + when(coordinator.getState(ANY_ID_2)).thenReturn(Optional.of(commitState)); + + executor = spy(executor); + doReturn(ANY_TIME_MILLIS_4).when(executor).getCommittedAt(); + + // Act + RecoveryExecutor.Result result = + executor.execute(snapshotKey, selection, transactionResult, ANY_ID_3); + + // Wait for recovery to complete + result.recoveryFuture.get(); + + // Assert + assertThat(result.recoveredResult).hasValue(prepareRolledForwardResult()); + verify(recovery).recover(eq(selection), eq(transactionResult), eq(Optional.of(commitState))); + } + + @Test + public void execute_CoordinatorStateIsCommitted_RecordWithDeletedState_ShouldCommit() + throws Exception { + // Arrange + TransactionResult transactionResult = prepareResult(TransactionState.DELETED); + Coordinator.State commitState = new Coordinator.State(ANY_ID_2, TransactionState.COMMITTED); + when(coordinator.getState(ANY_ID_2)).thenReturn(Optional.of(commitState)); + + executor = spy(executor); + + // Act + RecoveryExecutor.Result result = + executor.execute(snapshotKey, selection, transactionResult, ANY_ID_3); + + // Wait for recovery to complete + result.recoveryFuture.get(); + + // Assert + assertThat(result.recoveredResult).isNotPresent(); + verify(recovery).recover(eq(selection), eq(transactionResult), eq(Optional.of(commitState))); + } +} diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/RecoveryHandlerTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/RecoveryHandlerTest.java index 9408e15916..626795292e 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/RecoveryHandlerTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/RecoveryHandlerTest.java @@ -1,22 +1,32 @@ package com.scalar.db.transaction.consensuscommit; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; import com.scalar.db.api.DistributedStorage; +import com.scalar.db.api.Mutation; import com.scalar.db.api.Selection; import com.scalar.db.api.TableMetadata; import com.scalar.db.api.TransactionState; import com.scalar.db.common.ResultImpl; +import com.scalar.db.exception.storage.ExecutionException; +import com.scalar.db.exception.storage.NoMutationException; import com.scalar.db.io.Column; import com.scalar.db.io.DataType; import com.scalar.db.io.TextValue; import com.scalar.db.util.ScalarDbUtils; +import java.util.Collections; +import java.util.List; import java.util.Optional; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -24,6 +34,7 @@ import org.mockito.MockitoAnnotations; public class RecoveryHandlerTest { + private static final String ANY_NAME_1 = "name1"; private static final String ANY_TEXT_1 = "text1"; private static final String ANY_ID_1 = "id1"; @@ -71,47 +82,139 @@ private TransactionResult preparePreparedResult(long preparedAt) { @Test public void recover_SelectionAndResultGivenWhenCoordinatorStateCommitted_ShouldRollforward() - throws CoordinatorException { + throws CoordinatorException, ExecutionException { // Arrange TransactionResult result = preparePreparedResult(ANY_TIME_1); - when(coordinator.getState(ANY_ID_1)) - .thenReturn(Optional.of(new Coordinator.State(ANY_ID_1, TransactionState.COMMITTED))); + Optional state = + Optional.of(new Coordinator.State(ANY_ID_1, TransactionState.COMMITTED)); doNothing().when(handler).rollforwardRecord(any(Selection.class), any(TransactionResult.class)); // Act - handler.recover(selection, result); + handler.recover(selection, result, state); // Assert verify(handler).rollforwardRecord(selection, result); } + @Test + public void + recover_SelectionAndResultGivenWhenCoordinatorStateCommitted_NoMutationExceptionThrownByStorage_ShouldNotThrowAnyException() + throws ExecutionException { + // Arrange + TransactionResult result = preparePreparedResult(ANY_TIME_1); + Optional state = + Optional.of(new Coordinator.State(ANY_ID_1, TransactionState.COMMITTED)); + CommitMutationComposer composer = mock(CommitMutationComposer.class); + List mutations = Collections.singletonList(mock(Mutation.class)); + doReturn(mutations).when(composer).get(); + doReturn(composer).when(handler).createCommitMutationComposer(selection, result); + doThrow(NoMutationException.class).when(storage).mutate(any()); + + // Act Assert + assertThatCode(() -> handler.recover(selection, result, state)).doesNotThrowAnyException(); + + verify(handler).rollforwardRecord(selection, result); + verify(composer).get(); + verify(storage).mutate(mutations); + } + + @Test + public void + recover_SelectionAndResultGivenWhenCoordinatorStateCommitted_ExecutionExceptionThrownByStorage_ShouldThrowExecutionException() + throws ExecutionException { + // Arrange + TransactionResult result = preparePreparedResult(ANY_TIME_1); + Optional state = + Optional.of(new Coordinator.State(ANY_ID_1, TransactionState.COMMITTED)); + CommitMutationComposer composer = mock(CommitMutationComposer.class); + List mutations = Collections.singletonList(mock(Mutation.class)); + doReturn(mutations).when(composer).get(); + doReturn(composer).when(handler).createCommitMutationComposer(selection, result); + doThrow(ExecutionException.class).when(storage).mutate(any()); + + // Act Assert + assertThatThrownBy(() -> handler.recover(selection, result, state)) + .isInstanceOf(ExecutionException.class); + + verify(handler).rollforwardRecord(selection, result); + verify(composer).get(); + verify(storage).mutate(mutations); + } + @Test public void recover_SelectionAndResultGivenWhenCoordinatorStateAborted_ShouldRollback() - throws CoordinatorException { + throws CoordinatorException, ExecutionException { // Arrange TransactionResult result = preparePreparedResult(ANY_TIME_1); - when(coordinator.getState(ANY_ID_1)) - .thenReturn(Optional.of(new Coordinator.State(ANY_ID_1, TransactionState.ABORTED))); + Optional state = + Optional.of(new Coordinator.State(ANY_ID_1, TransactionState.ABORTED)); doNothing().when(handler).rollbackRecord(any(Selection.class), any(TransactionResult.class)); // Act - handler.recover(selection, result); + handler.recover(selection, result, state); // Assert verify(handler).rollbackRecord(selection, result); } + @Test + public void + recover_SelectionAndResultGivenWhenCoordinatorStateAborted_NoMutationExceptionThrownByStorage_ShouldNotThrowAnyException() + throws CoordinatorException, ExecutionException { + // Arrange + TransactionResult result = preparePreparedResult(ANY_TIME_1); + Optional state = + Optional.of(new Coordinator.State(ANY_ID_1, TransactionState.ABORTED)); + RollbackMutationComposer composer = mock(RollbackMutationComposer.class); + List mutations = Collections.singletonList(mock(Mutation.class)); + doReturn(mutations).when(composer).get(); + doReturn(composer).when(handler).createRollbackMutationComposer(selection, result); + doThrow(NoMutationException.class).when(storage).mutate(any()); + + // Act + assertThatCode(() -> handler.recover(selection, result, state)).doesNotThrowAnyException(); + + // Assert + verify(handler).rollbackRecord(selection, result); + verify(composer).get(); + verify(storage).mutate(mutations); + } + + @Test + public void + recover_SelectionAndResultGivenWhenCoordinatorStateAborted_ExecutionExceptionThrownByStorage_ShouldThrowExecutionException() + throws CoordinatorException, ExecutionException { + // Arrange + TransactionResult result = preparePreparedResult(ANY_TIME_1); + Optional state = + Optional.of(new Coordinator.State(ANY_ID_1, TransactionState.ABORTED)); + RollbackMutationComposer composer = mock(RollbackMutationComposer.class); + List mutations = Collections.singletonList(mock(Mutation.class)); + doReturn(mutations).when(composer).get(); + doReturn(composer).when(handler).createRollbackMutationComposer(selection, result); + doThrow(ExecutionException.class).when(storage).mutate(any()); + + // Act + assertThatThrownBy(() -> handler.recover(selection, result, state)) + .isInstanceOf(ExecutionException.class); + + // Assert + verify(handler).rollbackRecord(selection, result); + verify(composer).get(); + verify(storage).mutate(mutations); + } + @Test public void recover_SelectionAndResultGivenWhenCoordinatorStateNotExistsAndNotExpired_ShouldDoNothing() - throws CoordinatorException { + throws CoordinatorException, ExecutionException { // Arrange TransactionResult result = preparePreparedResult(System.currentTimeMillis()); - when(coordinator.getState(ANY_ID_1)).thenReturn(Optional.empty()); + Optional state = Optional.empty(); doNothing().when(handler).rollbackRecord(any(Selection.class), any(TransactionResult.class)); // Act - handler.recover(selection, result); + handler.recover(selection, result, state); // Assert verify(coordinator, never()) @@ -121,20 +224,63 @@ public void recover_SelectionAndResultGivenWhenCoordinatorStateAborted_ShouldRol @Test public void recover_SelectionAndResultGivenWhenCoordinatorStateNotExistsAndExpired_ShouldAbort() - throws CoordinatorException { + throws CoordinatorException, ExecutionException { // Arrange TransactionResult result = preparePreparedResult( System.currentTimeMillis() - RecoveryHandler.TRANSACTION_LIFETIME_MILLIS * 2); - when(coordinator.getState(ANY_ID_1)).thenReturn(Optional.empty()); + Optional state = Optional.empty(); doNothing().when(coordinator).putState(any(Coordinator.State.class)); doNothing().when(handler).rollbackRecord(any(Selection.class), any(TransactionResult.class)); // Act - handler.recover(selection, result); + handler.recover(selection, result, state); // Assert verify(coordinator).putStateForLazyRecoveryRollback(ANY_ID_1); verify(handler).rollbackRecord(selection, result); } + + @Test + public void + recover_SelectionAndResultGivenWhenCoordinatorStateNotExistsAndExpired_CoordinatorConflictExceptionByCoordinator_ShouldNotThrowAnyException() + throws CoordinatorException, ExecutionException { + // Arrange + TransactionResult result = + preparePreparedResult( + System.currentTimeMillis() - RecoveryHandler.TRANSACTION_LIFETIME_MILLIS * 2); + Optional state = Optional.empty(); + doThrow(CoordinatorConflictException.class) + .when(coordinator) + .putStateForLazyRecoveryRollback(anyString()); + + // Act + assertThatCode(() -> handler.recover(selection, result, state)).doesNotThrowAnyException(); + + // Assert + verify(coordinator).putStateForLazyRecoveryRollback(ANY_ID_1); + verify(handler, never()).rollbackRecord(selection, result); + } + + @Test + public void + recover_SelectionAndResultGivenWhenCoordinatorStateNotExistsAndExpired_CoordinatorExceptionByCoordinator_ShouldThrowCoordinatorException() + throws CoordinatorException, ExecutionException { + // Arrange + TransactionResult result = + preparePreparedResult( + System.currentTimeMillis() - RecoveryHandler.TRANSACTION_LIFETIME_MILLIS * 2); + Optional state = Optional.empty(); + doThrow(CoordinatorException.class) + .when(coordinator) + .putStateForLazyRecoveryRollback(anyString()); + + // Act + assertThatThrownBy(() -> handler.recover(selection, result, state)) + .isInstanceOf(CoordinatorException.class); + + // Assert + verify(coordinator).putStateForLazyRecoveryRollback(ANY_ID_1); + verify(handler, never()).rollbackRecord(selection, result); + } } diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposerTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposerTest.java index a848cbe324..b62ff3f228 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposerTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposerTest.java @@ -393,7 +393,6 @@ private List> extractAfterColumns(TransactionResult result) { public void add_GetAndPreparedResultByThisGiven_ShouldComposePut() throws ExecutionException { // Arrange TransactionResult result = prepareResult(TransactionState.PREPARED); - when(storage.get(any(Get.class))).thenReturn(Optional.of(result)); Get get = prepareGet(); // Act @@ -436,14 +435,12 @@ public void add_GetAndPreparedResultByThisGiven_ShouldComposePut() throws Execut .timestampTZValue(BEFORE_PREFIX + ANY_NAME_13, null) .build(); assertThat(actual).isEqualTo(expected); - verify(storage).get(any(Get.class)); } @Test public void add_GetAndDeletedResultByThisGiven_ShouldComposePut() throws ExecutionException { // Arrange TransactionResult result = prepareResult(TransactionState.DELETED); - when(storage.get(any(Get.class))).thenReturn(Optional.of(result)); Get get = prepareGet(); // Act @@ -485,7 +482,54 @@ public void add_GetAndDeletedResultByThisGiven_ShouldComposePut() throws Executi .timestampTZValue(BEFORE_PREFIX + ANY_NAME_13, null) .build(); assertThat(actual).isEqualTo(expected); - verify(storage).get(any(Get.class)); + } + + @Test + public void add_GetAndPreparedResultWithNullMetadataByThisGiven_ShouldComposePut() + throws ExecutionException { + // Arrange + TransactionResult result = prepareResultWithNullMetadata(TransactionState.PREPARED); + Get get = prepareGet(); + + // Act + composer.add(get, result); + + // Assert + Put actual = (Put) composer.get().get(0); + PutBuilder.Buildable builder = + Put.newBuilder() + .namespace(get.forNamespace().get()) + .table(get.forTable().get()) + .partitionKey(get.getPartitionKey()) + .clusteringKey(get.getClusteringKey().get()) + .consistency(Consistency.LINEARIZABLE) + .condition( + ConditionBuilder.putIf(ConditionBuilder.column(ID).isEqualToText(ANY_ID_2)) + .and( + ConditionBuilder.column(STATE) + .isEqualToInt(TransactionState.PREPARED.get())) + .build()); + extractAfterColumns(prepareInitialResultWithNullMetadata()).forEach(builder::value); + Put expected = + builder + .textValue(BEFORE_ID, null) + .intValue(BEFORE_STATE, null) + .intValue(BEFORE_VERSION, null) + .bigIntValue(BEFORE_PREPARED_AT, null) + .bigIntValue(BEFORE_COMMITTED_AT, null) + .intValue(BEFORE_PREFIX + ANY_NAME_3, null) + .booleanValue(BEFORE_PREFIX + ANY_NAME_4, null) + .bigIntValue(BEFORE_PREFIX + ANY_NAME_5, null) + .floatValue(BEFORE_PREFIX + ANY_NAME_6, null) + .doubleValue(BEFORE_PREFIX + ANY_NAME_7, null) + .textValue(BEFORE_PREFIX + ANY_NAME_8, null) + .blobValue(BEFORE_PREFIX + ANY_NAME_9, (byte[]) null) + .dateValue(BEFORE_PREFIX + ANY_NAME_10, null) + .timeValue(BEFORE_PREFIX + ANY_NAME_11, null) + .timestampValue(BEFORE_PREFIX + ANY_NAME_12, null) + .timestampTZValue(BEFORE_PREFIX + ANY_NAME_13, null) + .build(); + assertThat(actual).isEqualTo(expected); } @Test @@ -658,7 +702,6 @@ public void add_PutAndResultFromSnapshotGivenAndItsAlreadyRollbackDeleted_Should public void add_ScanAndPreparedResultByThisGiven_ShouldComposePut() throws ExecutionException { // Arrange TransactionResult result = prepareResult(TransactionState.PREPARED); - when(storage.get(any(Get.class))).thenReturn(Optional.of(result)); Scan scan = prepareScan(); // Act @@ -701,14 +744,12 @@ public void add_ScanAndPreparedResultByThisGiven_ShouldComposePut() throws Execu .timestampTZValue(BEFORE_PREFIX + ANY_NAME_13, null) .build(); assertThat(actual).isEqualTo(expected); - verify(storage).get(any(Get.class)); } @Test public void add_ScanAndDeletedResultByThisGiven_ShouldComposePut() throws ExecutionException { // Arrange TransactionResult result = prepareResult(TransactionState.DELETED); - when(storage.get(any(Get.class))).thenReturn(Optional.of(result)); Scan scan = prepareScan(); // Act @@ -750,7 +791,6 @@ public void add_ScanAndDeletedResultByThisGiven_ShouldComposePut() throws Execut .timestampTZValue(BEFORE_PREFIX + ANY_NAME_13, null) .build(); assertThat(actual).isEqualTo(expected); - verify(storage).get(any(Get.class)); } @Test @@ -758,7 +798,6 @@ public void add_ScanAndPreparedResultByThisGivenAndBeforeResultNotGiven_ShouldCo throws ExecutionException { // Arrange TransactionResult result = prepareInitialResult(ANY_ID_2, TransactionState.PREPARED); - when(storage.get(any(Get.class))).thenReturn(Optional.of(result)); Scan scan = prepareScan(); // Act @@ -777,73 +816,5 @@ public void add_ScanAndPreparedResultByThisGivenAndBeforeResultNotGiven_ShouldCo new ConditionalExpression( STATE, toStateValue(TransactionState.PREPARED), Operator.EQ))); assertThat(actual).isEqualTo(expected); - verify(storage).get(any(Get.class)); - } - - @Test - public void add_GetAndPreparedResultWithNullMetadataByThisGiven_ShouldComposePut() - throws ExecutionException { - // Arrange - TransactionResult result = prepareResultWithNullMetadata(TransactionState.PREPARED); - when(storage.get(any(Get.class))).thenReturn(Optional.of(result)); - Get get = prepareGet(); - - // Act - composer.add(get, result); - - // Assert - Put actual = (Put) composer.get().get(0); - PutBuilder.Buildable builder = - Put.newBuilder() - .namespace(get.forNamespace().get()) - .table(get.forTable().get()) - .partitionKey(get.getPartitionKey()) - .clusteringKey(get.getClusteringKey().get()) - .consistency(Consistency.LINEARIZABLE) - .condition( - ConditionBuilder.putIf(ConditionBuilder.column(ID).isEqualToText(ANY_ID_2)) - .and( - ConditionBuilder.column(STATE) - .isEqualToInt(TransactionState.PREPARED.get())) - .build()); - extractAfterColumns(prepareInitialResultWithNullMetadata()).forEach(builder::value); - Put expected = - builder - .textValue(BEFORE_ID, null) - .intValue(BEFORE_STATE, null) - .intValue(BEFORE_VERSION, null) - .bigIntValue(BEFORE_PREPARED_AT, null) - .bigIntValue(BEFORE_COMMITTED_AT, null) - .intValue(BEFORE_PREFIX + ANY_NAME_3, null) - .booleanValue(BEFORE_PREFIX + ANY_NAME_4, null) - .bigIntValue(BEFORE_PREFIX + ANY_NAME_5, null) - .floatValue(BEFORE_PREFIX + ANY_NAME_6, null) - .doubleValue(BEFORE_PREFIX + ANY_NAME_7, null) - .textValue(BEFORE_PREFIX + ANY_NAME_8, null) - .blobValue(BEFORE_PREFIX + ANY_NAME_9, (byte[]) null) - .dateValue(BEFORE_PREFIX + ANY_NAME_10, null) - .timeValue(BEFORE_PREFIX + ANY_NAME_11, null) - .timestampValue(BEFORE_PREFIX + ANY_NAME_12, null) - .timestampTZValue(BEFORE_PREFIX + ANY_NAME_13, null) - .build(); - assertThat(actual).isEqualTo(expected); - verify(storage).get(any(Get.class)); - } - - @Test - public void add_GetAndInitialResultWithNullMetadataGivenFromStorage_ShouldDoNothing() - throws ExecutionException { - // Arrange - TransactionResult obtainedResult = prepareResultWithNullMetadata(TransactionState.PREPARED); - TransactionResult currentResult = prepareInitialResultWithNullMetadata(); - when(storage.get(any(Get.class))).thenReturn(Optional.of(currentResult)); - Get get = prepareGet(); - - // Act - composer.add(get, obtainedResult); - - // Assert - assertThat(composer.get().size()).isEqualTo(0); - verify(storage).get(any(Get.class)); } } diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManagerTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManagerTest.java index a0c0b62992..f5528bfd76 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManagerTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManagerTest.java @@ -62,7 +62,7 @@ public class TwoPhaseConsensusCommitManagerTest { @Mock private DatabaseConfig databaseConfig; @Mock private Coordinator coordinator; @Mock private ParallelExecutor parallelExecutor; - @Mock private RecoveryHandler recovery; + @Mock private RecoveryExecutor recoveryExecutor; @Mock private CommitHandler commit; private TwoPhaseConsensusCommitManager manager; @@ -82,7 +82,7 @@ public void setUp() throws Exception { databaseConfig, coordinator, parallelExecutor, - recovery, + recoveryExecutor, commit); } @@ -127,9 +127,6 @@ public void begin_CalledTwice_ReturnRespectiveConsensusCommitWithSharedObjects() assertThat(transaction1.getCommitHandler()) .isEqualTo(transaction2.getCommitHandler()) .isEqualTo(commit); - assertThat(transaction1.getRecoveryHandler()) - .isEqualTo(transaction2.getRecoveryHandler()) - .isEqualTo(recovery); } @Test @@ -206,9 +203,6 @@ public void start_CalledTwice_ReturnRespectiveConsensusCommitWithSharedObjects() assertThat(transaction1.getCommitHandler()) .isEqualTo(transaction2.getCommitHandler()) .isEqualTo(commit); - assertThat(transaction1.getRecoveryHandler()) - .isEqualTo(transaction2.getRecoveryHandler()) - .isEqualTo(recovery); } @Test diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitTest.java index a2fd4d4867..36ef01e5c6 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitTest.java @@ -59,7 +59,6 @@ public class TwoPhaseConsensusCommitTest { @Mock private Snapshot snapshot; @Mock private CrudHandler crud; @Mock private CommitHandler commit; - @Mock private RecoveryHandler recovery; @Mock private ConsensusCommitMutationOperationChecker mutationOperationChecker; private TwoPhaseConsensusCommit transaction; @@ -67,8 +66,8 @@ public class TwoPhaseConsensusCommitTest { public void setUp() throws Exception { MockitoAnnotations.openMocks(this).close(); - // Arrange - transaction = new TwoPhaseConsensusCommit(crud, commit, recovery, mutationOperationChecker); + // Arrange1 + transaction = new TwoPhaseConsensusCommit(crud, commit, mutationOperationChecker); when(crud.areAllScannersClosed()).thenReturn(true); } @@ -116,28 +115,9 @@ public void get_GetGiven_ShouldCallCrudHandlerGet() throws CrudException { // Assert assertThat(actual).isPresent(); - verify(recovery, never()).recover(get, result); verify(crud).get(get); } - @Test - public void get_GetForUncommittedRecordGiven_ShouldRecoverRecord() throws CrudException { - // Arrange - Get get = prepareGet(); - TransactionResult result = mock(TransactionResult.class); - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - when(crud.get(get)).thenThrow(toThrow); - when(crud.getSnapshot()).thenReturn(snapshot); - when(toThrow.getSelection()).thenReturn(get); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - // Act - assertThatThrownBy(() -> transaction.get(get)).isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery).recover(get, result); - } - @Test public void scan_ScanGiven_ShouldCallCrudHandlerScan() throws CrudException { // Arrange @@ -155,22 +135,6 @@ public void scan_ScanGiven_ShouldCallCrudHandlerScan() throws CrudException { verify(crud).scan(scan); } - @Test - public void scan_ScanForUncommittedRecordGiven_ShouldRecoverRecord() throws CrudException { - // Arrange - Scan scan = prepareScan(); - TransactionResult result = mock(TransactionResult.class); - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - when(crud.scan(scan)).thenThrow(toThrow); - when(toThrow.getSelection()).thenReturn(scan); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - // Act Assert - assertThatThrownBy(() -> transaction.scan(scan)).isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(scan, result); - } - @Test public void getScannerAndScannerOne_ShouldCallCrudHandlerGetScannerAndScannerOne() throws CrudException { @@ -191,29 +155,6 @@ public void getScannerAndScannerOne_ShouldCallCrudHandlerGetScannerAndScannerOne verify(scanner).one(); } - @Test - public void - getScannerAndScannerOne_UncommittedRecordExceptionThrownByScannerOne_ShouldRecoverRecord() - throws CrudException { - // Arrange - Scan scan = prepareScan(); - - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - TransactionResult result = mock(TransactionResult.class); - when(toThrow.getSelection()).thenReturn(scan); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); - when(scanner.one()).thenThrow(toThrow); - when(crud.getScanner(scan)).thenReturn(scanner); - - // Act Assert - TransactionCrudOperable.Scanner actualScanner = transaction.getScanner(scan); - assertThatThrownBy(actualScanner::one).isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(scan, result); - } - @Test public void getScannerAndScannerAll_ShouldCallCrudHandlerGetScannerAndScannerAll() throws CrudException { @@ -235,29 +176,6 @@ public void getScannerAndScannerAll_ShouldCallCrudHandlerGetScannerAndScannerAll verify(scanner).all(); } - @Test - public void - getScannerAndScannerAll_UncommittedRecordExceptionThrownByScannerAll_ShouldRecoverRecord() - throws CrudException { - // Arrange - Scan scan = prepareScan(); - - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - TransactionResult result = mock(TransactionResult.class); - when(toThrow.getSelection()).thenReturn(scan); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - TransactionCrudOperable.Scanner scanner = mock(TransactionCrudOperable.Scanner.class); - when(scanner.all()).thenThrow(toThrow); - when(crud.getScanner(scan)).thenReturn(scanner); - - // Act Assert - TransactionCrudOperable.Scanner actualScanner = transaction.getScanner(scan); - assertThatThrownBy(actualScanner::all).isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(scan, result); - } - @Test public void put_PutGiven_ShouldCallCrudHandlerPut() throws ExecutionException, CrudException { // Arrange @@ -287,25 +205,6 @@ public void put_TwoPutsGiven_ShouldCallCrudHandlerPutTwice() verify(mutationOperationChecker, times(2)).check(put); } - @Test - public void put_PutGivenAndUncommittedRecordExceptionThrown_ShouldRecoverRecord() - throws CrudException { - // Arrange - Put put = preparePut(); - Get get = prepareGet(); - - TransactionResult result = mock(TransactionResult.class); - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - doThrow(toThrow).when(crud).put(put); - when(toThrow.getSelection()).thenReturn(get); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - // Act Assert - assertThatThrownBy(() -> transaction.put(put)).isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(get, result); - } - @Test public void delete_DeleteGiven_ShouldCallCrudHandlerDelete() throws CrudException, ExecutionException { @@ -336,26 +235,6 @@ public void delete_TwoDeletesGiven_ShouldCallCrudHandlerDeleteTwice() verify(mutationOperationChecker, times(2)).check(delete); } - @Test - public void delete_DeleteGivenAndUncommittedRecordExceptionThrown_ShouldRecoverRecord() - throws CrudException { - // Arrange - Delete delete = prepareDelete(); - Get get = prepareGet(); - - TransactionResult result = mock(TransactionResult.class); - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - doThrow(toThrow).when(crud).delete(delete); - when(toThrow.getSelection()).thenReturn(get); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - // Act Assert - assertThatThrownBy(() -> transaction.delete(delete)) - .isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(get, result); - } - @Test public void insert_InsertGiven_ShouldCallCrudHandlerPut() throws CrudException, ExecutionException { @@ -416,47 +295,6 @@ public void upsert_UpsertGiven_ShouldCallCrudHandlerPut() verify(mutationOperationChecker).check(expectedPut); } - @Test - public void upsert_UpsertForUncommittedRecordGiven_ShouldRecoverRecord() throws CrudException { - // Arrange - Upsert upsert = - Upsert.newBuilder() - .namespace(ANY_NAMESPACE) - .table(ANY_TABLE_NAME) - .partitionKey(Key.ofText(ANY_NAME_1, ANY_TEXT_1)) - .clusteringKey(Key.ofText(ANY_NAME_2, ANY_TEXT_2)) - .textValue(ANY_NAME_3, ANY_TEXT_3) - .build(); - Put put = - Put.newBuilder() - .namespace(ANY_NAMESPACE) - .table(ANY_TABLE_NAME) - .partitionKey(Key.ofText(ANY_NAME_1, ANY_TEXT_1)) - .clusteringKey(Key.ofText(ANY_NAME_2, ANY_TEXT_2)) - .textValue(ANY_NAME_3, ANY_TEXT_3) - .enableImplicitPreRead() - .build(); - Get get = - Get.newBuilder() - .namespace(ANY_NAMESPACE) - .table(ANY_TABLE_NAME) - .partitionKey(Key.ofText(ANY_NAME_1, ANY_TEXT_1)) - .clusteringKey(Key.ofText(ANY_NAME_2, ANY_TEXT_2)) - .build(); - - TransactionResult result = mock(TransactionResult.class); - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - doThrow(toThrow).when(crud).put(put); - when(toThrow.getSelection()).thenReturn(get); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - // Act Assert - assertThatThrownBy(() -> transaction.upsert(upsert)) - .isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(get, result); - } - @Test public void update_UpdateWithoutConditionGiven_ShouldCallCrudHandlerPut() throws CrudException, ExecutionException { @@ -645,48 +483,6 @@ public void update_UpdateWithConditionGiven_ShouldCallCrudHandlerPut() .hasMessageNotContaining("PutIfExists"); } - @Test - public void update_UpdateForUncommittedRecordGiven_ShouldRecoverRecord() throws CrudException { - // Arrange - Update update = - Update.newBuilder() - .namespace(ANY_NAMESPACE) - .table(ANY_TABLE_NAME) - .partitionKey(Key.ofText(ANY_NAME_1, ANY_TEXT_1)) - .clusteringKey(Key.ofText(ANY_NAME_2, ANY_TEXT_2)) - .textValue(ANY_NAME_3, ANY_TEXT_3) - .build(); - Put put = - Put.newBuilder() - .namespace(ANY_NAMESPACE) - .table(ANY_TABLE_NAME) - .partitionKey(Key.ofText(ANY_NAME_1, ANY_TEXT_1)) - .clusteringKey(Key.ofText(ANY_NAME_2, ANY_TEXT_2)) - .textValue(ANY_NAME_3, ANY_TEXT_3) - .condition(ConditionBuilder.putIfExists()) - .enableImplicitPreRead() - .build(); - Get get = - Get.newBuilder() - .namespace(ANY_NAMESPACE) - .table(ANY_TABLE_NAME) - .partitionKey(Key.ofText(ANY_NAME_1, ANY_TEXT_1)) - .clusteringKey(Key.ofText(ANY_NAME_2, ANY_TEXT_2)) - .build(); - - TransactionResult result = mock(TransactionResult.class); - UncommittedRecordException toThrow = mock(UncommittedRecordException.class); - doThrow(toThrow).when(crud).put(put); - when(toThrow.getSelection()).thenReturn(get); - when(toThrow.getResults()).thenReturn(Collections.singletonList(result)); - - // Act Assert - assertThatThrownBy(() -> transaction.update(update)) - .isInstanceOf(UncommittedRecordException.class); - - verify(recovery).recover(get, result); - } - @Test public void mutate_PutAndDeleteGiven_ShouldCallCrudHandlerPutAndDelete() throws CrudException, ExecutionException { @@ -715,7 +511,9 @@ public void prepare_ProcessedCrudGiven_ShouldPrepareRecordsWithSnapshot() transaction.prepare(); // Assert + verify(crud).areAllScannersClosed(); verify(crud).readIfImplicitPreReadEnabled(); + verify(crud).waitForRecoveryCompletionIfNecessary(); verify(commit).prepareRecords(snapshot); } @@ -731,28 +529,6 @@ public void prepare_ProcessedCrudGiven_ShouldPrepareRecordsWithSnapshot() assertThatThrownBy(transaction::prepare).isInstanceOf(PreparationConflictException.class); } - @Test - public void - prepare_ProcessedCrudGiven_UncommittedRecordExceptionThrownWhileImplicitPreRead_ShouldPerformLazyRecoveryAndThrowPreparationConflictException() - throws CrudException { - // Arrange - when(crud.getSnapshot()).thenReturn(snapshot); - - Get get = mock(Get.class); - TransactionResult result = mock(TransactionResult.class); - - UncommittedRecordException uncommittedRecordException = mock(UncommittedRecordException.class); - when(uncommittedRecordException.getSelection()).thenReturn(get); - when(uncommittedRecordException.getResults()).thenReturn(Collections.singletonList(result)); - - doThrow(uncommittedRecordException).when(crud).readIfImplicitPreReadEnabled(); - - // Act Assert - assertThatThrownBy(transaction::prepare).isInstanceOf(PreparationConflictException.class); - - verify(recovery).recover(get, result); - } - @Test public void prepare_ProcessedCrudGiven_CrudExceptionThrownWhileImplicitPreRead_ShouldThrowPreparationException() @@ -774,6 +550,18 @@ public void prepare_ScannerNotClosed_ShouldThrowIllegalStateException() { assertThatThrownBy(() -> transaction.prepare()).isInstanceOf(IllegalStateException.class); } + @Test + public void + prepare_CrudExceptionThrownByCrudHandlerWaitForRecoveryCompletionIfNecessary_ShouldThrowPreparationException() + throws CrudException { + // Arrange + when(crud.getSnapshot()).thenReturn(snapshot); + doThrow(CrudException.class).when(crud).waitForRecoveryCompletionIfNecessary(); + + // Act Assert + assertThatThrownBy(() -> transaction.prepare()).isInstanceOf(PreparationException.class); + } + @Test public void validate_ProcessedCrudGiven_ShouldValidateRecordsWithSnapshot() throws ValidationException, PreparationException { diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitNullMetadataIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitNullMetadataIntegrationTestBase.java index 67c5e4f204..60ebed7dc0 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitNullMetadataIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitNullMetadataIntegrationTestBase.java @@ -6,7 +6,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import com.scalar.db.api.Consistency; @@ -75,6 +74,7 @@ public abstract class ConsensusCommitNullMetadataIntegrationTestBase { private DistributedStorage storage; private Coordinator coordinator; private RecoveryHandler recovery; + private RecoveryExecutor recoveryExecutor; @Nullable private CoordinatorGroupCommitter groupCommitter; @BeforeAll @@ -138,6 +138,12 @@ public void setUp() throws Exception { TransactionTableMetadataManager tableMetadataManager = new TransactionTableMetadataManager(admin, -1); recovery = spy(new RecoveryHandler(storage, coordinator, tableMetadataManager)); + recoveryExecutor = + new RecoveryExecutor( + coordinator, + recovery, + tableMetadataManager, + consensusCommitConfig.getRecoveryExecutorCount()); groupCommitter = CoordinatorGroupCommitter.from(consensusCommitConfig).orElse(null); CommitHandler commit = spy(createCommitHandler(tableMetadataManager, groupCommitter)); manager = @@ -148,7 +154,7 @@ public void setUp() throws Exception { databaseConfig, coordinator, parallelExecutor, - recovery, + recoveryExecutor, commit, groupCommitter); } @@ -183,6 +189,7 @@ public void afterAll() throws Exception { consensusCommitAdmin.close(); originalStorage.close(); parallelExecutor.close(); + recoveryExecutor.close(); } private void dropTables() throws ExecutionException { @@ -489,19 +496,6 @@ private void selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_Sh DistributedTransaction transaction = manager.begin(); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery).rollforwardRecord(any(Selection.class), any(TransactionResult.class)); TransactionResult result; if (s instanceof Get) { Optional r = transaction.get((Get) s); @@ -512,12 +506,20 @@ private void selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_Sh assertThat(results.size()).isEqualTo(1); result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); } - transaction.commit(); assertThat(result.getId()).isEqualTo(ANY_ID_1); Assertions.assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); assertThat(result.getVersion()).isEqualTo(1); assertThat(result.getCommittedAt()).isGreaterThan(0); + + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).getCrudHandler().waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollforwardRecord(any(Selection.class), any(TransactionResult.class)); } @Test @@ -543,19 +545,6 @@ private void selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_Shou DistributedTransaction transaction = manager.begin(); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); TransactionResult result; if (s instanceof Get) { Optional r = transaction.get((Get) s); @@ -566,12 +555,20 @@ private void selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_Shou assertThat(results.size()).isEqualTo(1); result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); } - transaction.commit(); assertThat(result.getId()).isNull(); Assertions.assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); assertThat(result.getVersion()).isEqualTo(0); assertThat(result.getCommittedAt()).isEqualTo(0); + + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).getCrudHandler().waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); } @Test @@ -608,12 +605,12 @@ public void scan_ScanGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( }) .isInstanceOf(UncommittedRecordException.class); + transaction.rollback(); + // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); + verify(recovery, never()).recover(any(Selection.class), any(TransactionResult.class), any()); verify(recovery, never()).rollbackRecord(any(Selection.class), any(TransactionResult.class)); verify(coordinator, never()).putState(any(Coordinator.State.class)); - - transaction.commit(); } @Test @@ -644,20 +641,6 @@ public void scan_ScanGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( DistributedTransaction transaction = manager.begin(); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); - verify(coordinator).putState(new Coordinator.State(ANY_ID_1, TransactionState.ABORTED)); - verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); TransactionResult result; if (s instanceof Get) { Optional r = transaction.get((Get) s); @@ -668,183 +651,37 @@ public void scan_ScanGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( assertThat(results.size()).isEqualTo(1); result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); } - transaction.commit(); assertThat(result.getId()).isNull(); Assertions.assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); assertThat(result.getVersion()).isEqualTo(0); assertThat(result.getCommittedAt()).isEqualTo(0); - } - @Test - public void get_GetGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() - throws ExecutionException, CoordinatorException, TransactionException { - Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - get); - } - - @Test - public void - scan_ScanGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - scan); - } - - private void - selection_SelectionGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - Selection s) throws ExecutionException, CoordinatorException, TransactionException { - // Arrange - long current = System.currentTimeMillis(); - populatePreparedRecordWithNullMetadataAndCoordinatorStateRecord( - storage, - namespace1, - TABLE_1, - TransactionState.PREPARED, - current, - TransactionState.COMMITTED); - - ConsensusCommit transaction = (ConsensusCommit) manager.begin(); - - transaction.setBeforeRecoveryHook( - () -> - assertThatThrownBy( - () -> { - DistributedTransaction another = manager.begin(); - if (s instanceof Get) { - another.get((Get) s); - } else { - another.scan((Scan) s); - } - another.commit(); - }) - .isInstanceOf(UncommittedRecordException.class)); - - // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery, times(2)).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery, times(2)) - .rollforwardRecord(any(Selection.class), any(TransactionResult.class)); - TransactionResult result; - if (s instanceof Get) { - Optional r = transaction.get((Get) s); - assertThat(r).isPresent(); - result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); - } else { - List results = transaction.scan((Scan) s); - assertThat(results.size()).isEqualTo(1); - result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); - } transaction.commit(); - assertThat(result.getId()).isEqualTo(ANY_ID_1); - Assertions.assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); - assertThat(result.getVersion()).isEqualTo(1); - assertThat(result.getCommittedAt()).isGreaterThan(0); - } - - @Test - public void - get_GetGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly() - throws ExecutionException, CoordinatorException, TransactionException { - Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - get); - } - - @Test - public void - scan_ScanGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - scan); - } - - private void - selection_SelectionGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - Selection s) throws ExecutionException, CoordinatorException, TransactionException { - // Arrange - long current = System.currentTimeMillis(); - populatePreparedRecordWithNullMetadataAndCoordinatorStateRecord( - storage, namespace1, TABLE_1, TransactionState.PREPARED, current, TransactionState.ABORTED); - - ConsensusCommit transaction = (ConsensusCommit) manager.begin(); - - transaction.setBeforeRecoveryHook( - () -> - assertThatThrownBy( - () -> { - DistributedTransaction another = manager.begin(); - if (s instanceof Get) { - another.get((Get) s); - } else { - another.scan((Scan) s); - } - another.commit(); - }) - .isInstanceOf(UncommittedRecordException.class)); - - // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); + // Wait for the recovery to complete + ((ConsensusCommit) transaction).getCrudHandler().waitForRecoveryCompletion(); // Assert - verify(recovery, times(2)).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery, times(2)).rollbackRecord(any(Selection.class), any(TransactionResult.class)); - // rollback called twice but executed once actually - TransactionResult result; - if (s instanceof Get) { - Optional r = transaction.get((Get) s); - assertThat(r).isPresent(); - result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); - } else { - List results = transaction.scan((Scan) s); - assertThat(results.size()).isEqualTo(1); - result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); - } - transaction.commit(); - - assertThat(result.getId()).isNull(); - Assertions.assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); - assertThat(result.getVersion()).isEqualTo(0); - assertThat(result.getCommittedAt()).isEqualTo(0); + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(coordinator).putState(new Coordinator.State(ANY_ID_1, TransactionState.ABORTED)); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); } @Test - public void - get_GetGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly() - throws ExecutionException, CoordinatorException, TransactionException { + public void get_GetGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() + throws ExecutionException, CoordinatorException, TransactionException { Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( + selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( get); } @Test public void - scan_ScanGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly() + scan_ScanGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() throws ExecutionException, CoordinatorException, TransactionException { Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( + selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( scan); } @@ -862,26 +699,21 @@ private void selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_Sho DistributedTransaction transaction = manager.begin(); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery).rollforwardRecord(any(Selection.class), any(TransactionResult.class)); if (s instanceof Get) { assertThat(transaction.get((Get) s).isPresent()).isFalse(); } else { List results = transaction.scan((Scan) s); assertThat(results.size()).isEqualTo(0); } + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).getCrudHandler().waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollforwardRecord(any(Selection.class), any(TransactionResult.class)); } @Test @@ -907,19 +739,6 @@ private void selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_Shoul DistributedTransaction transaction = manager.begin(); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); TransactionResult result; if (s instanceof Get) { Optional r = transaction.get((Get) s); @@ -930,12 +749,20 @@ private void selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_Shoul assertThat(results.size()).isEqualTo(1); result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); } - transaction.commit(); assertThat(result.getId()).isNull(); Assertions.assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); assertThat(result.getVersion()).isEqualTo(0); assertThat(result.getCommittedAt()).isEqualTo(0); + + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).getCrudHandler().waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); } @Test @@ -972,12 +799,12 @@ public void scan_ScanGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback() }) .isInstanceOf(UncommittedRecordException.class); + transaction.rollback(); + // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); + verify(recovery, never()).recover(any(Selection.class), any(TransactionResult.class), any()); verify(recovery, never()).rollbackRecord(any(Selection.class), any(TransactionResult.class)); verify(coordinator, never()).putState(any(Coordinator.State.class)); - - transaction.commit(); } @Test @@ -1008,20 +835,6 @@ public void scan_ScanGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback() DistributedTransaction transaction = manager.begin(); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); - verify(coordinator).putState(new Coordinator.State(ANY_ID_1, TransactionState.ABORTED)); - verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); TransactionResult result; if (s instanceof Get) { Optional r = transaction.get((Get) s); @@ -1032,174 +845,37 @@ public void scan_ScanGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback() assertThat(results.size()).isEqualTo(1); result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); } - transaction.commit(); assertThat(result.getId()).isNull(); Assertions.assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); assertThat(result.getVersion()).isEqualTo(0); assertThat(result.getCommittedAt()).isEqualTo(0); - } - - @Test - public void get_GetGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() - throws ExecutionException, CoordinatorException, TransactionException { - Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - get); - } - @Test - public void - scan_ScanGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - scan); - } - - private void - selection_SelectionGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - Selection s) throws ExecutionException, CoordinatorException, TransactionException { - // Arrange - long current = System.currentTimeMillis(); - populatePreparedRecordWithNullMetadataAndCoordinatorStateRecord( - storage, - namespace1, - TABLE_1, - TransactionState.DELETED, - current, - TransactionState.COMMITTED); - - ConsensusCommit transaction = (ConsensusCommit) manager.begin(); - - transaction.setBeforeRecoveryHook( - () -> - assertThatThrownBy( - () -> { - DistributedTransaction another = manager.begin(); - if (s instanceof Get) { - another.get((Get) s); - } else { - another.scan((Scan) s); - } - another.commit(); - }) - .isInstanceOf(UncommittedRecordException.class)); - - // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery, times(2)).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery, times(2)) - .rollforwardRecord(any(Selection.class), any(TransactionResult.class)); - if (s instanceof Get) { - assertThat(transaction.get((Get) s).isPresent()).isFalse(); - } else { - List results = transaction.scan((Scan) s); - assertThat(results.size()).isEqualTo(0); - } transaction.commit(); - } - @Test - public void - get_GetGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly() - throws ExecutionException, CoordinatorException, TransactionException { - Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - get); - } - - @Test - public void - scan_ScanGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - scan); - } - - private void - selection_SelectionGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - Selection s) throws ExecutionException, CoordinatorException, TransactionException { - // Arrange - long current = System.currentTimeMillis(); - populatePreparedRecordWithNullMetadataAndCoordinatorStateRecord( - storage, namespace1, TABLE_1, TransactionState.DELETED, current, TransactionState.ABORTED); - - ConsensusCommit transaction = (ConsensusCommit) manager.begin(); - - transaction.setBeforeRecoveryHook( - () -> - assertThatThrownBy( - () -> { - DistributedTransaction another = manager.begin(); - if (s instanceof Get) { - another.get((Get) s); - } else { - another.scan((Scan) s); - } - another.commit(); - }) - .isInstanceOf(UncommittedRecordException.class)); - - // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); + // Wait for the recovery to complete + ((ConsensusCommit) transaction).getCrudHandler().waitForRecoveryCompletion(); // Assert - verify(recovery, times(2)).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery, times(2)).rollbackRecord(any(Selection.class), any(TransactionResult.class)); - // rollback called twice but executed once actually - TransactionResult result; - if (s instanceof Get) { - Optional r = transaction.get((Get) s); - assertThat(r).isPresent(); - result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); - } else { - List results = transaction.scan((Scan) s); - assertThat(results.size()).isEqualTo(1); - result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); - } - transaction.commit(); - - assertThat(result.getId()).isNull(); - Assertions.assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); - assertThat(result.getVersion()).isEqualTo(0); - assertThat(result.getCommittedAt()).isEqualTo(0); + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(coordinator).putState(new Coordinator.State(ANY_ID_1, TransactionState.ABORTED)); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); } @Test - public void - get_GetGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly() - throws ExecutionException, CoordinatorException, TransactionException { + public void get_GetGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() + throws ExecutionException, CoordinatorException, TransactionException { Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( get); } @Test public void - scan_ScanGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly() + scan_ScanGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() throws ExecutionException, CoordinatorException, TransactionException { Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( scan); } @@ -1433,15 +1109,6 @@ public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateAborted_ShouldRoll selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback(scanAll); } - @Test - public void - scanAll_ScanAllGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - scanAll); - } - @Test public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward() throws ExecutionException, CoordinatorException, TransactionException { @@ -1449,15 +1116,6 @@ public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateCommitted_ShouldRo selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward(scanAll); } - @Test - public void - scanAll_ScanAllGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - scanAll); - } - @Test public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() @@ -1483,15 +1141,6 @@ public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateAborted_ShouldRol selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback(scanAll); } - @Test - public void - scanAll_ScanAllGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - scanAll); - } - @Test public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward() throws ExecutionException, CoordinatorException, TransactionException { @@ -1499,15 +1148,6 @@ public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateCommitted_ShouldR selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward(scanAll); } - @Test - public void - scanAll_ScanAllGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly() - throws ExecutionException, CoordinatorException, TransactionException { - Scan scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - scanAll); - } - @Test public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java index 13c2073c26..cfff4e913c 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java @@ -66,6 +66,7 @@ import java.util.Set; import java.util.UUID; import java.util.stream.IntStream; +import java.util.stream.Stream; import javax.annotation.Nullable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -75,7 +76,9 @@ import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.condition.EnabledIf; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public abstract class ConsensusCommitSpecificIntegrationTestBase { @@ -109,6 +112,7 @@ public abstract class ConsensusCommitSpecificIntegrationTestBase { private DistributedStorage storage; private Coordinator coordinator; private RecoveryHandler recovery; + private RecoveryExecutor recoveryExecutor; private CommitHandler commit; @Nullable private CoordinatorGroupCommitter groupCommitter; @@ -175,6 +179,12 @@ public void setUp() throws Exception { TransactionTableMetadataManager tableMetadataManager = new TransactionTableMetadataManager(admin, -1); recovery = spy(new RecoveryHandler(storage, coordinator, tableMetadataManager)); + recoveryExecutor = + new RecoveryExecutor( + coordinator, + recovery, + tableMetadataManager, + consensusCommitConfig.getRecoveryExecutorCount()); groupCommitter = CoordinatorGroupCommitter.from(consensusCommitConfig).orElse(null); commit = spy(createCommitHandler(tableMetadataManager, groupCommitter)); manager = @@ -185,7 +195,7 @@ public void setUp() throws Exception { databaseConfig, coordinator, parallelExecutor, - recovery, + recoveryExecutor, commit, groupCommitter); } @@ -203,6 +213,7 @@ private CommitHandler createCommitHandler( @AfterEach public void tearDown() { + recoveryExecutor.close(); if (groupCommitter != null) { groupCommitter.close(); } @@ -338,8 +349,16 @@ public enum CommitType { DELAYED_GROUP_COMMIT } + static Stream commitTypeAndIsolation() { + return Arrays.stream(CommitType.values()) + .flatMap( + commitType -> + Arrays.stream(Isolation.values()) + .map(isolation -> Arguments.of(commitType, isolation))); + } + private void selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward( - Selection s, CommitType commitType) + Selection s, CommitType commitType, Isolation isolation, boolean useScanner) throws ExecutionException, CoordinatorException, TransactionException { // Arrange long current = System.currentTimeMillis(); @@ -352,60 +371,74 @@ private void selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_Sh current, TransactionState.COMMITTED, commitType); - DistributedTransaction transaction = manager.begin(); + DistributedTransaction transaction = manager.begin(isolation); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery).rollforwardRecord(any(Selection.class), any(TransactionResult.class)); TransactionResult result; if (s instanceof Get) { Optional r = transaction.get((Get) s); assertThat(r).isPresent(); result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); } else { - List results = transaction.scan((Scan) s); + List results; + if (!useScanner) { + results = transaction.scan((Scan) s); + } else { + try (TransactionCrudOperable.Scanner scanner = transaction.getScanner((Scan) s)) { + results = scanner.all(); + } + } assertThat(results.size()).isEqualTo(1); result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); } - transaction.commit(); assertThat(result.getId()).isEqualTo(ongoingTxId); assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); assertThat(result.getVersion()).isEqualTo(2); assertThat(result.getCommittedAt()).isGreaterThan(0); + + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).getCrudHandler().waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollforwardRecord(any(Selection.class), any(TransactionResult.class)); } - @ParameterizedTest() - @EnumSource(CommitType.class) + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") public void get_GetGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward( - CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { Get get = prepareGet(0, 0, namespace1, TABLE_1); selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward( - get, commitType); + get, commitType, isolation, false); } - @ParameterizedTest() - @EnumSource(CommitType.class) + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") public void scan_ScanGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward( - CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { + Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); + selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward( + scan, commitType, isolation, false); + } + + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void getScanner_ScanGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward( - scan, commitType); + scan, commitType, isolation, true); } private void selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( - Selection s, CommitType commitType) + Selection s, CommitType commitType, Isolation isolation, boolean useScanner) throws ExecutionException, CoordinatorException, TransactionException { // Arrange long current = System.currentTimeMillis(); @@ -417,65 +450,81 @@ private void selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_Shou current, TransactionState.ABORTED, commitType); - DistributedTransaction transaction = manager.begin(); + DistributedTransaction transaction = manager.begin(isolation); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); TransactionResult result; if (s instanceof Get) { Optional r = transaction.get((Get) s); assertThat(r).isPresent(); result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); } else { - List results = transaction.scan((Scan) s); + List results; + if (!useScanner) { + results = transaction.scan((Scan) s); + } else { + try (TransactionCrudOperable.Scanner scanner = transaction.getScanner((Scan) s)) { + results = scanner.all(); + } + } assertThat(results.size()).isEqualTo(1); result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); } - transaction.commit(); assertThat(result.getId()).isEqualTo(ANY_ID_1); assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); assertThat(result.getVersion()).isEqualTo(1); assertThat(result.getCommittedAt()).isEqualTo(1); + + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).getCrudHandler().waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); } - @ParameterizedTest() - @EnumSource(CommitType.class) + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") public void get_GetGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( - CommitType commitType) throws TransactionException, ExecutionException, CoordinatorException { + CommitType commitType, Isolation isolation) + throws TransactionException, ExecutionException, CoordinatorException { Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback(get, commitType); + selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( + get, commitType, isolation, false); } - @ParameterizedTest() - @EnumSource(CommitType.class) + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") public void scan_ScanGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( - CommitType commitType) throws TransactionException, ExecutionException, CoordinatorException { + CommitType commitType, Isolation isolation) + throws TransactionException, ExecutionException, CoordinatorException { + Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); + selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( + scan, commitType, isolation, false); + } + + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void getScanner_ScanGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( + CommitType commitType, Isolation isolation) + throws TransactionException, ExecutionException, CoordinatorException { Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback(scan, commitType); + selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( + scan, commitType, isolation, true); } private void selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( - Selection s, CommitType commitType) + Selection s, CommitType commitType, Isolation isolation, boolean useScanner) throws ExecutionException, CoordinatorException, TransactionException { // Arrange long prepared_at = System.currentTimeMillis(); populatePreparedRecordAndCoordinatorStateRecord( storage, namespace1, TABLE_1, TransactionState.PREPARED, prepared_at, null, commitType); - DistributedTransaction transaction = manager.begin(); + DistributedTransaction transaction = manager.begin(isolation); // Act assertThatThrownBy( @@ -483,292 +532,294 @@ public void scan_ScanGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( if (s instanceof Get) { transaction.get((Get) s); } else { - transaction.scan((Scan) s); + if (!useScanner) { + transaction.scan((Scan) s); + } else { + try (TransactionCrudOperable.Scanner scanner = transaction.getScanner((Scan) s)) { + scanner.all(); + } + } } }) .isInstanceOf(UncommittedRecordException.class); + transaction.rollback(); + // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); + verify(recovery, never()).recover(any(Selection.class), any(TransactionResult.class), any()); verify(recovery, never()).rollbackRecord(any(Selection.class), any(TransactionResult.class)); verify(coordinator, never()).putState(any(Coordinator.State.class)); - - transaction.commit(); } - @ParameterizedTest() - @EnumSource(CommitType.class) + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") public void get_GetGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( - CommitType commitType) + CommitType commitType, Isolation isolation) throws ExecutionException, CoordinatorException, TransactionException { Get get = prepareGet(0, 0, namespace1, TABLE_1); selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( - get, commitType); + get, commitType, isolation, false); } - @ParameterizedTest() - @EnumSource(CommitType.class) + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") public void scan_ScanGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( - CommitType commitType) + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { + Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); + selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + scan, commitType, isolation, false); + } + + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void + getScanner_ScanGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + CommitType commitType, Isolation isolation) throws ExecutionException, CoordinatorException, TransactionException { Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( - scan, commitType); + scan, commitType, isolation, true); } private void selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - Selection s, CommitType commitType) + Selection s, CommitType commitType, Isolation isolation, boolean useScanner) throws ExecutionException, CoordinatorException, TransactionException { // Arrange long prepared_at = System.currentTimeMillis() - RecoveryHandler.TRANSACTION_LIFETIME_MILLIS - 1; String ongoingTxId = populatePreparedRecordAndCoordinatorStateRecord( storage, namespace1, TABLE_1, TransactionState.PREPARED, prepared_at, null, commitType); - DistributedTransaction transaction = manager.begin(); + DistributedTransaction transaction = manager.begin(isolation); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); - verify(coordinator).putState(new Coordinator.State(ongoingTxId, TransactionState.ABORTED)); - verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); TransactionResult result; if (s instanceof Get) { Optional r = transaction.get((Get) s); assertThat(r).isPresent(); result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); } else { - List results = transaction.scan((Scan) s); + List results; + if (!useScanner) { + results = transaction.scan((Scan) s); + } else { + try (TransactionCrudOperable.Scanner scanner = transaction.getScanner((Scan) s)) { + results = scanner.all(); + } + } assertThat(results.size()).isEqualTo(1); result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); } - transaction.commit(); assertThat(result.getId()).isEqualTo(ANY_ID_1); assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); assertThat(result.getVersion()).isEqualTo(1); assertThat(result.getCommittedAt()).isEqualTo(1); + + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).getCrudHandler().waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(coordinator).putState(new Coordinator.State(ongoingTxId, TransactionState.ABORTED)); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); } - @ParameterizedTest() - @EnumSource(CommitType.class) + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") public void get_GetGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { Get get = prepareGet(0, 0, namespace1, TABLE_1); selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - get, commitType); + get, commitType, isolation, false); } - @ParameterizedTest() - @EnumSource(CommitType.class) + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") public void scan_ScanGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - CommitType commitType) + CommitType commitType, Isolation isolation) throws ExecutionException, CoordinatorException, TransactionException { Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - scan, commitType); + scan, commitType, isolation, false); } - private void - selection_SelectionGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - Selection s, CommitType commitType) + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void + getScanner_ScanGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + CommitType commitType, Isolation isolation) throws ExecutionException, CoordinatorException, TransactionException { + Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); + selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + scan, commitType, isolation, true); + } + + private void selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( + Selection s, CommitType commitType, Isolation isolation, boolean useScanner) + throws ExecutionException, CoordinatorException, TransactionException { // Arrange long current = System.currentTimeMillis(); - String ongoingTxId = - populatePreparedRecordAndCoordinatorStateRecord( - storage, - namespace1, - TABLE_1, - TransactionState.PREPARED, - current, - TransactionState.COMMITTED, - commitType); - - ConsensusCommit transaction = (ConsensusCommit) manager.begin(); - - transaction.setBeforeRecoveryHook( - () -> - assertThatThrownBy( - () -> { - DistributedTransaction another = manager.begin(); - if (s instanceof Get) { - another.get((Get) s); - } else { - another.scan((Scan) s); - } - another.commit(); - }) - .isInstanceOf(UncommittedRecordException.class)); + populatePreparedRecordAndCoordinatorStateRecord( + storage, + namespace1, + TABLE_1, + TransactionState.DELETED, + current, + TransactionState.COMMITTED, + commitType); + DistributedTransaction transaction = manager.begin(isolation); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery, times(2)).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery, times(2)) - .rollforwardRecord(any(Selection.class), any(TransactionResult.class)); - TransactionResult result; if (s instanceof Get) { - Optional r = transaction.get((Get) s); - assertThat(r).isPresent(); - result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); + assertThat(transaction.get((Get) s).isPresent()).isFalse(); } else { - List results = transaction.scan((Scan) s); - assertThat(results.size()).isEqualTo(1); - result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); + List results; + if (!useScanner) { + results = transaction.scan((Scan) s); + } else { + try (TransactionCrudOperable.Scanner scanner = transaction.getScanner((Scan) s)) { + results = scanner.all(); + } + } + assertThat(results.size()).isEqualTo(0); } + transaction.commit(); - assertThat(result.getId()).isEqualTo(ongoingTxId); - assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); - assertThat(result.getVersion()).isEqualTo(2); - assertThat(result.getCommittedAt()).isGreaterThan(0); + // Wait for the recovery to complete + ((ConsensusCommit) transaction).getCrudHandler().waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollforwardRecord(any(Selection.class), any(TransactionResult.class)); } - @ParameterizedTest() - @EnumSource(CommitType.class) - public void - get_GetGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - CommitType commitType) - throws ExecutionException, CoordinatorException, TransactionException { + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void get_GetGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - get, commitType); + selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( + get, commitType, isolation, false); } - @ParameterizedTest() - @EnumSource(CommitType.class) - public void - scan_ScanGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - CommitType commitType) - throws ExecutionException, CoordinatorException, TransactionException { + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void scan_ScanGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - scan, commitType); + selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( + scan, commitType, isolation, false); } - private void - selection_SelectionGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - Selection s, CommitType commitType) - throws ExecutionException, CoordinatorException, TransactionException { + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void getScanner_ScanGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { + Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); + selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( + scan, commitType, isolation, true); + } + + private void selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( + Selection s, CommitType commitType, Isolation isolation, boolean useScanner) + throws ExecutionException, CoordinatorException, TransactionException { // Arrange long current = System.currentTimeMillis(); populatePreparedRecordAndCoordinatorStateRecord( storage, namespace1, TABLE_1, - TransactionState.PREPARED, + TransactionState.DELETED, current, TransactionState.ABORTED, commitType); - - ConsensusCommit transaction = (ConsensusCommit) manager.begin(); - - transaction.setBeforeRecoveryHook( - () -> - assertThatThrownBy( - () -> { - DistributedTransaction another = manager.begin(); - if (s instanceof Get) { - another.get((Get) s); - } else { - another.scan((Scan) s); - } - another.commit(); - }) - .isInstanceOf(UncommittedRecordException.class)); + DistributedTransaction transaction = manager.begin(isolation); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery, times(2)).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery, times(2)).rollbackRecord(any(Selection.class), any(TransactionResult.class)); - // rollback called twice but executed once actually TransactionResult result; if (s instanceof Get) { Optional r = transaction.get((Get) s); assertThat(r).isPresent(); result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); } else { - List results = transaction.scan((Scan) s); + List results; + if (!useScanner) { + results = transaction.scan((Scan) s); + } else { + try (TransactionCrudOperable.Scanner scanner = transaction.getScanner((Scan) s)) { + results = scanner.all(); + } + } assertThat(results.size()).isEqualTo(1); result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); } - transaction.commit(); assertThat(result.getId()).isEqualTo(ANY_ID_1); assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); assertThat(result.getVersion()).isEqualTo(1); assertThat(result.getCommittedAt()).isEqualTo(1); + + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).getCrudHandler().waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); } - @ParameterizedTest() - @EnumSource(CommitType.class) - public void - get_GetGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - CommitType commitType) - throws ExecutionException, CoordinatorException, TransactionException { + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void get_GetGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - get, commitType); + selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( + get, commitType, isolation, false); } - @ParameterizedTest() - @EnumSource(CommitType.class) - public void - scan_ScanGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - CommitType commitType) - throws ExecutionException, CoordinatorException, TransactionException { + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void scan_ScanGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - scan, commitType); + selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( + scan, commitType, isolation, false); } - private void selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( - Selection s, CommitType commitType) + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void getScanner_ScanGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( + CommitType commitType, Isolation isolation) throws ExecutionException, CoordinatorException, TransactionException { + Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); + selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( + scan, commitType, isolation, true); + } + + private void + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + Selection s, CommitType commitType, Isolation isolation, boolean useScanner) + throws ExecutionException, CoordinatorException, TransactionException { // Arrange - long current = System.currentTimeMillis(); + long prepared_at = System.currentTimeMillis(); populatePreparedRecordAndCoordinatorStateRecord( - storage, - namespace1, - TABLE_1, - TransactionState.DELETED, - current, - TransactionState.COMMITTED, - commitType); - DistributedTransaction transaction = manager.begin(); + storage, namespace1, TABLE_1, TransactionState.DELETED, prepared_at, null, commitType); + DistributedTransaction transaction = manager.begin(isolation); // Act assertThatThrownBy( @@ -776,220 +827,330 @@ private void selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_Sho if (s instanceof Get) { transaction.get((Get) s); } else { - transaction.scan((Scan) s); + if (!useScanner) { + transaction.scan((Scan) s); + } else { + try (TransactionCrudOperable.Scanner scanner = transaction.getScanner((Scan) s)) { + scanner.all(); + } + } } }) .isInstanceOf(UncommittedRecordException.class); + transaction.rollback(); + // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery).rollforwardRecord(any(Selection.class), any(TransactionResult.class)); - if (s instanceof Get) { - assertThat(transaction.get((Get) s).isPresent()).isFalse(); - } else { - List results = transaction.scan((Scan) s); - assertThat(results.size()).isEqualTo(0); - } - transaction.commit(); + verify(recovery, never()).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery, never()).rollbackRecord(any(Selection.class), any(TransactionResult.class)); + verify(coordinator, never()).putState(any(Coordinator.State.class)); } - @ParameterizedTest() - @EnumSource(CommitType.class) - public void get_GetGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( - CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void + get_GetGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( - get, commitType); + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + get, commitType, isolation, false); } - @ParameterizedTest() - @EnumSource(CommitType.class) - public void scan_ScanGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( - CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void + scan_ScanGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( - scan, commitType); + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + scan, commitType, isolation, false); } - private void selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( - Selection s, CommitType commitType) - throws ExecutionException, CoordinatorException, TransactionException { + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void + getScanner_ScanGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { + Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + scan, commitType, isolation, true); + } + + private void + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + Selection s, CommitType commitType, Isolation isolation, boolean useScanner) + throws ExecutionException, CoordinatorException, TransactionException { // Arrange - long current = System.currentTimeMillis(); - populatePreparedRecordAndCoordinatorStateRecord( - storage, - namespace1, - TABLE_1, - TransactionState.DELETED, - current, - TransactionState.ABORTED, - commitType); - DistributedTransaction transaction = manager.begin(); + long prepared_at = System.currentTimeMillis() - RecoveryHandler.TRANSACTION_LIFETIME_MILLIS - 1; + String ongoingTxId = + populatePreparedRecordAndCoordinatorStateRecord( + storage, namespace1, TABLE_1, TransactionState.DELETED, prepared_at, null, commitType); + DistributedTransaction transaction = manager.begin(isolation); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); TransactionResult result; if (s instanceof Get) { Optional r = transaction.get((Get) s); assertThat(r).isPresent(); result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); } else { - List results = transaction.scan((Scan) s); + List results; + if (!useScanner) { + results = transaction.scan((Scan) s); + } else { + try (TransactionCrudOperable.Scanner scanner = transaction.getScanner((Scan) s)) { + results = scanner.all(); + } + } assertThat(results.size()).isEqualTo(1); result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); } - transaction.commit(); assertThat(result.getId()).isEqualTo(ANY_ID_1); assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); assertThat(result.getVersion()).isEqualTo(1); assertThat(result.getCommittedAt()).isEqualTo(1); + + transaction.commit(); + + // Wait for the recovery to complete + ((ConsensusCommit) transaction).getCrudHandler().waitForRecoveryCompletion(); + + // Assert + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(coordinator).putState(new Coordinator.State(ongoingTxId, TransactionState.ABORTED)); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); } - @ParameterizedTest() - @EnumSource(CommitType.class) - public void get_GetGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( - CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void get_GetGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback(get, commitType); + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + get, commitType, isolation, false); } - @ParameterizedTest() - @EnumSource(CommitType.class) - public void scan_ScanGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( - CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void scan_ScanGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback(scan, commitType); + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + scan, commitType, isolation, false); } - private void - selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( - Selection s, CommitType commitType) + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void + getScanner_ScanGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { + Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + scan, commitType, isolation, true); + } + + @ParameterizedTest + @EnumSource(CommitType.class) + public void + update_UpdateGivenForPreparedWhenCoordinatorStateCommitted_ShouldUpdateAfterRollforward( + CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { // Arrange - long prepared_at = System.currentTimeMillis(); + long current = System.currentTimeMillis(); populatePreparedRecordAndCoordinatorStateRecord( - storage, namespace1, TABLE_1, TransactionState.DELETED, prepared_at, null, commitType); + storage, + namespace1, + TABLE_1, + TransactionState.PREPARED, + current, + TransactionState.COMMITTED, + commitType); + + Get get = + Get.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .build(); + DistributedTransaction transaction = manager.begin(); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); + Optional result = transaction.get(get); + assertThat(result.isPresent()).isTrue(); + assertThat(result.get().getInt(BALANCE)).isEqualTo(NEW_BALANCE); + + int expectedBalance = result.get().getInt(BALANCE) + 100; + + transaction.update( + Update.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .intValue(BALANCE, expectedBalance) + .build()); + + transaction.commit(); // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery, never()).rollbackRecord(any(Selection.class), any(TransactionResult.class)); - verify(coordinator, never()).putState(any(Coordinator.State.class)); + Optional actual = manager.get(get); + assertThat(actual.isPresent()).isTrue(); + assertThat(actual.get().getInt(BALANCE)).isEqualTo(expectedBalance); + + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollforwardRecord(any(Selection.class), any(TransactionResult.class)); + } + + @ParameterizedTest + @EnumSource(CommitType.class) + public void update_UpdateGivenForPreparedWhenCoordinatorStateAborted_ShouldUpdateAfterRollback( + CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { + // Arrange + long current = System.currentTimeMillis(); + populatePreparedRecordAndCoordinatorStateRecord( + storage, + namespace1, + TABLE_1, + TransactionState.PREPARED, + current, + TransactionState.ABORTED, + commitType); + + Get get = + Get.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .build(); + + DistributedTransaction transaction = manager.begin(); + + // Act + Optional result = transaction.get(get); + assertThat(result.isPresent()).isTrue(); + assertThat(result.get().getInt(BALANCE)).isEqualTo(INITIAL_BALANCE); + + int expectedBalance = result.get().getInt(BALANCE) + 100; + + transaction.update( + Update.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .intValue(BALANCE, expectedBalance) + .build()); transaction.commit(); + + // Assert + Optional actual = manager.get(get); + assertThat(actual.isPresent()).isTrue(); + assertThat(actual.get().getInt(BALANCE)).isEqualTo(expectedBalance); + + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); } - @ParameterizedTest() + @ParameterizedTest @EnumSource(CommitType.class) public void - get_GetGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + update_UpdateGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldThrowUncommittedRecordException( CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { - Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( - get, commitType); + // Arrange + long prepared_at = System.currentTimeMillis(); + populatePreparedRecordAndCoordinatorStateRecord( + storage, namespace1, TABLE_1, TransactionState.PREPARED, prepared_at, null, commitType); + + DistributedTransaction transaction = manager.begin(); + + // Act + assertThatThrownBy( + () -> + transaction.update( + Update.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .intValue(BALANCE, 100) + .build())) + .isInstanceOf(UncommittedRecordException.class); + + transaction.rollback(); + + // Assert + verify(recovery, never()).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery, never()).rollbackRecord(any(Selection.class), any(TransactionResult.class)); + verify(coordinator, never()).putState(any(Coordinator.State.class)); } - @ParameterizedTest() + @ParameterizedTest @EnumSource(CommitType.class) public void - scan_ScanGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + update_UpdateGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldUpdateAfterAbortTransaction( CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { - Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( - scan, commitType); - } - - private void - selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - Selection s, CommitType commitType) - throws ExecutionException, CoordinatorException, TransactionException { // Arrange long prepared_at = System.currentTimeMillis() - RecoveryHandler.TRANSACTION_LIFETIME_MILLIS - 1; String ongoingTxId = populatePreparedRecordAndCoordinatorStateRecord( - storage, namespace1, TABLE_1, TransactionState.DELETED, prepared_at, null, commitType); + storage, namespace1, TABLE_1, TransactionState.PREPARED, prepared_at, null, commitType); + + Get get = + Get.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .build(); + DistributedTransaction transaction = manager.begin(); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); + Optional result = transaction.get(get); + assertThat(result.isPresent()).isTrue(); + assertThat(result.get().getInt(BALANCE)).isEqualTo(INITIAL_BALANCE); + + int expectedBalance = result.get().getInt(BALANCE) + 100; + + transaction.update( + Update.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .intValue(BALANCE, expectedBalance) + .build()); - // Assert - verify(recovery).recover(any(Selection.class), any(TransactionResult.class)); - verify(coordinator).putState(new Coordinator.State(ongoingTxId, TransactionState.ABORTED)); - verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); - TransactionResult result; - if (s instanceof Get) { - Optional r = transaction.get((Get) s); - assertThat(r).isPresent(); - result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); - } else { - List results = transaction.scan((Scan) s); - assertThat(results.size()).isEqualTo(1); - result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); - } transaction.commit(); - assertThat(result.getId()).isEqualTo(ANY_ID_1); - assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); - assertThat(result.getVersion()).isEqualTo(1); - assertThat(result.getCommittedAt()).isEqualTo(1); - } + // Assert + Optional actual = manager.get(get); + assertThat(actual.isPresent()).isTrue(); + assertThat(actual.get().getInt(BALANCE)).isEqualTo(expectedBalance); - @ParameterizedTest() - @EnumSource(CommitType.class) - public void get_GetGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { - Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - get, commitType); + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(coordinator).putState(new Coordinator.State(ongoingTxId, TransactionState.ABORTED)); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); } - @ParameterizedTest() + @ParameterizedTest @EnumSource(CommitType.class) - public void scan_ScanGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { - Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - scan, commitType); - } - - private void - selection_SelectionGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - Selection s, CommitType commitType) + public void + insert_InsertGivenForDeletedWhenCoordinatorStateCommitted_ShouldInsertAfterRollforward( + CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { // Arrange long current = System.currentTimeMillis(); @@ -1002,72 +1163,46 @@ public void scan_ScanGivenForDeletedWhenCoordinatorStateNotExistAndExpired_Shoul TransactionState.COMMITTED, commitType); - ConsensusCommit transaction = (ConsensusCommit) manager.begin(); - - transaction.setBeforeRecoveryHook( - () -> - assertThatThrownBy( - () -> { - DistributedTransaction another = manager.begin(); - if (s instanceof Get) { - another.get((Get) s); - } else { - another.scan((Scan) s); - } - another.commit(); - }) - .isInstanceOf(UncommittedRecordException.class)); + Get get = + Get.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .build(); + + DistributedTransaction transaction = manager.begin(); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); + Optional result = transaction.get(get); + assertThat(result.isPresent()).isFalse(); + + int expectedBalance = 100; + + transaction.insert( + Insert.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .intValue(BALANCE, expectedBalance) + .build()); + + transaction.commit(); // Assert - verify(recovery, times(2)).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery, times(2)) - .rollforwardRecord(any(Selection.class), any(TransactionResult.class)); - if (s instanceof Get) { - assertThat(transaction.get((Get) s).isPresent()).isFalse(); - } else { - List results = transaction.scan((Scan) s); - assertThat(results.size()).isEqualTo(0); - } - transaction.commit(); - } + Optional actual = manager.get(get); + assertThat(actual.isPresent()).isTrue(); + assertThat(actual.get().getInt(BALANCE)).isEqualTo(expectedBalance); - @ParameterizedTest() - @EnumSource(CommitType.class) - public void - get_GetGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - CommitType commitType) - throws ExecutionException, CoordinatorException, TransactionException { - Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - get, commitType); + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollforwardRecord(any(Selection.class), any(TransactionResult.class)); } - @ParameterizedTest() + @ParameterizedTest @EnumSource(CommitType.class) - public void - scan_ScanGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - CommitType commitType) - throws ExecutionException, CoordinatorException, TransactionException { - Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - scan, commitType); - } - - private void - selection_SelectionGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - Selection s, CommitType commitType) - throws ExecutionException, CoordinatorException, TransactionException { + public void update_UpdateGivenForDeletedWhenCoordinatorStateAborted_ShouldUpdateAfterRollback( + CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { // Arrange long current = System.currentTimeMillis(); populatePreparedRecordAndCoordinatorStateRecord( @@ -1079,75 +1214,125 @@ public void scan_ScanGivenForDeletedWhenCoordinatorStateNotExistAndExpired_Shoul TransactionState.ABORTED, commitType); - ConsensusCommit transaction = (ConsensusCommit) manager.begin(); - - transaction.setBeforeRecoveryHook( - () -> - assertThatThrownBy( - () -> { - DistributedTransaction another = manager.begin(); - if (s instanceof Get) { - another.get((Get) s); - } else { - another.scan((Scan) s); - } - another.commit(); - }) - .isInstanceOf(UncommittedRecordException.class)); + Get get = + Get.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .build(); + + DistributedTransaction transaction = manager.begin(); // Act - assertThatThrownBy( - () -> { - if (s instanceof Get) { - transaction.get((Get) s); - } else { - transaction.scan((Scan) s); - } - }) - .isInstanceOf(UncommittedRecordException.class); + Optional result = transaction.get(get); + assertThat(result.isPresent()).isTrue(); + assertThat(result.get().getInt(BALANCE)).isEqualTo(INITIAL_BALANCE); + + int expectedBalance = result.get().getInt(BALANCE) + 100; + + transaction.update( + Update.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .intValue(BALANCE, expectedBalance) + .build()); - // Assert - verify(recovery, times(2)).recover(any(Selection.class), any(TransactionResult.class)); - verify(recovery, times(2)).rollbackRecord(any(Selection.class), any(TransactionResult.class)); - // rollback called twice but executed once actually - TransactionResult result; - if (s instanceof Get) { - Optional r = transaction.get((Get) s); - assertThat(r).isPresent(); - result = (TransactionResult) ((FilteredResult) r.get()).getOriginalResult(); - } else { - List results = transaction.scan((Scan) s); - assertThat(results.size()).isEqualTo(1); - result = (TransactionResult) ((FilteredResult) results.get(0)).getOriginalResult(); - } transaction.commit(); - assertThat(result.getId()).isEqualTo(ANY_ID_1); - assertThat(result.getState()).isEqualTo(TransactionState.COMMITTED); - assertThat(result.getVersion()).isEqualTo(1); - assertThat(result.getCommittedAt()).isEqualTo(1); + // Assert + Optional actual = manager.get(get); + assertThat(actual.isPresent()).isTrue(); + assertThat(actual.get().getInt(BALANCE)).isEqualTo(expectedBalance); + + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); } - @ParameterizedTest() + @ParameterizedTest @EnumSource(CommitType.class) public void - get_GetGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( + update_UpdateGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldThrowUncommittedRecordException( CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { - Get get = prepareGet(0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - get, commitType); + // Arrange + long prepared_at = System.currentTimeMillis(); + populatePreparedRecordAndCoordinatorStateRecord( + storage, namespace1, TABLE_1, TransactionState.DELETED, prepared_at, null, commitType); + + DistributedTransaction transaction = manager.begin(); + + // Act + assertThatThrownBy( + () -> + transaction.update( + Update.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .intValue(BALANCE, 100) + .build())) + .isInstanceOf(UncommittedRecordException.class); + + transaction.rollback(); + + // Assert + verify(recovery, never()).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(recovery, never()).rollbackRecord(any(Selection.class), any(TransactionResult.class)); + verify(coordinator, never()).putState(any(Coordinator.State.class)); } - @ParameterizedTest() + @ParameterizedTest @EnumSource(CommitType.class) public void - scan_ScanGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( + update_UpdateGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldUpdateAfterAbortTransaction( CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { - Scan scan = prepareScan(0, 0, 0, namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - scan, commitType); + // Arrange + long prepared_at = System.currentTimeMillis() - RecoveryHandler.TRANSACTION_LIFETIME_MILLIS - 1; + String ongoingTxId = + populatePreparedRecordAndCoordinatorStateRecord( + storage, namespace1, TABLE_1, TransactionState.DELETED, prepared_at, null, commitType); + + Get get = + Get.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .build(); + + DistributedTransaction transaction = manager.begin(); + + // Act + Optional result = transaction.get(get); + assertThat(result.isPresent()).isTrue(); + assertThat(result.get().getInt(BALANCE)).isEqualTo(INITIAL_BALANCE); + + int expectedBalance = result.get().getInt(BALANCE) + 100; + + transaction.update( + Update.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .intValue(BALANCE, expectedBalance) + .build()); + + transaction.commit(); + + // Assert + Optional actual = manager.get(get); + assertThat(actual.isPresent()).isTrue(); + assertThat(actual.get().getInt(BALANCE)).isEqualTo(expectedBalance); + + verify(recovery).recover(any(Selection.class), any(TransactionResult.class), any()); + verify(coordinator).putState(new Coordinator.State(ongoingTxId, TransactionState.ABORTED)); + verify(recovery).rollbackRecord(any(Selection.class), any(TransactionResult.class)); } @Test @@ -2812,146 +2997,190 @@ public void scanAll_ScanAllGivenForCommittedRecord_ShouldReturnRecord() .isEqualTo(TransactionState.COMMITTED); } + @Test + public void scanAll_ScanAllGivenForNonExisting_ShouldReturnEmpty() throws TransactionException { + // Arrange + DistributedTransaction putTransaction = manager.begin(); + putTransaction.put(preparePut(0, 0, namespace1, TABLE_1)); + putTransaction.commit(); + + DistributedTransaction transaction = manager.begin(); + ScanAll scanAll = prepareScanAll(namespace2, TABLE_2); + + // Act + List results = transaction.scan(scanAll); + transaction.commit(); + + // Assert + assertThat(results.size()).isEqualTo(0); + } + @ParameterizedTest - @EnumSource(CommitType.class) + @MethodSource("commitTypeAndIsolation") public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( - CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( - scanAll, commitType); + scanAll, commitType, isolation, false); } @ParameterizedTest - @EnumSource(CommitType.class) - public void - scanAll_ScanAllGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - CommitType commitType) - throws ExecutionException, CoordinatorException, TransactionException { + @MethodSource("commitTypeAndIsolation") + public void getScanner_ScanAllGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - scanAll, commitType); + selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback( + scanAll, commitType, isolation, true); } @ParameterizedTest - @EnumSource(CommitType.class) + @MethodSource("commitTypeAndIsolation") public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( - CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( - scanAll, commitType); + scanAll, commitType, isolation, false); } @ParameterizedTest - @EnumSource(CommitType.class) + @MethodSource("commitTypeAndIsolation") + public void getScanner_ScanAllGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { + ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); + selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( + scanAll, commitType, isolation, true); + } + + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") public void - scanAll_ScanAllGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - CommitType commitType) + scanAll_ScanAllGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + CommitType commitType, Isolation isolation) throws ExecutionException, CoordinatorException, TransactionException { ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - scanAll, commitType); + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + scanAll, commitType, isolation, false); } @ParameterizedTest - @EnumSource(CommitType.class) + @MethodSource("commitTypeAndIsolation") public void - scanAll_ScanAllGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - CommitType commitType) + getScanner_ScanAllGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + CommitType commitType, Isolation isolation) throws ExecutionException, CoordinatorException, TransactionException { ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - scanAll, commitType); + scanAll, commitType, isolation, true); } @ParameterizedTest - @EnumSource(CommitType.class) + @MethodSource("commitTypeAndIsolation") public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( - CommitType commitType) + CommitType commitType, Isolation isolation) throws ExecutionException, CoordinatorException, TransactionException { ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( - scanAll, commitType); + scanAll, commitType, isolation, false); } - @Test - public void scanAll_ScanAllGivenForNonExisting_ShouldReturnEmpty() throws TransactionException { - // Arrange - DistributedTransaction putTransaction = manager.begin(); - putTransaction.put(preparePut(0, 0, namespace1, TABLE_1)); - putTransaction.commit(); - - DistributedTransaction transaction = manager.begin(); - ScanAll scanAll = prepareScanAll(namespace2, TABLE_2); - - // Act - List results = transaction.scan(scanAll); - transaction.commit(); - - // Assert - assertThat(results.size()).isEqualTo(0); + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void + getScanner_ScanAllGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { + ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); + selection_SelectionGivenForDeletedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + scanAll, commitType, isolation, true); } @ParameterizedTest - @EnumSource(CommitType.class) + @MethodSource("commitTypeAndIsolation") public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( - CommitType commitType) throws TransactionException, ExecutionException, CoordinatorException { + CommitType commitType, Isolation isolation) + throws TransactionException, ExecutionException, CoordinatorException { ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( - scanAll, commitType); + scanAll, commitType, isolation, false); } @ParameterizedTest - @EnumSource(CommitType.class) - public void - scanAll_ScanAllGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - CommitType commitType) - throws ExecutionException, CoordinatorException, TransactionException { + @MethodSource("commitTypeAndIsolation") + public void getScanner_ScanAllGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( + CommitType commitType, Isolation isolation) + throws TransactionException, ExecutionException, CoordinatorException { ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - scanAll, commitType); + selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( + scanAll, commitType, isolation, true); } @ParameterizedTest - @EnumSource(CommitType.class) + @MethodSource("commitTypeAndIsolation") public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward( - CommitType commitType) throws ExecutionException, CoordinatorException, TransactionException { + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward( - scanAll, commitType); + scanAll, commitType, isolation, false); } @ParameterizedTest - @EnumSource(CommitType.class) + @MethodSource("commitTypeAndIsolation") + public void getScanner_ScanAllGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward( + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { + ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); + selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward( + scanAll, commitType, isolation, true); + } + + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") public void - scanAll_ScanAllGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - CommitType commitType) + scanAll_ScanAllGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + CommitType commitType, Isolation isolation) throws ExecutionException, CoordinatorException, TransactionException { ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); - selection_SelectionGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - scanAll, commitType); + selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + scanAll, commitType, isolation, false); } @ParameterizedTest - @EnumSource(CommitType.class) + @MethodSource("commitTypeAndIsolation") public void - scanAll_ScanAllGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - CommitType commitType) + getScanner_ScanAllGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( + CommitType commitType, Isolation isolation) throws ExecutionException, CoordinatorException, TransactionException { ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction( - scanAll, commitType); + scanAll, commitType, isolation, true); } @ParameterizedTest - @EnumSource(CommitType.class) + @MethodSource("commitTypeAndIsolation") public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( - CommitType commitType) + CommitType commitType, Isolation isolation) + throws ExecutionException, CoordinatorException, TransactionException { + ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); + selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + scanAll, commitType, isolation, false); + } + + @ParameterizedTest + @MethodSource("commitTypeAndIsolation") + public void + getScanner_ScanAllGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( + CommitType commitType, Isolation isolation) throws ExecutionException, CoordinatorException, TransactionException { ScanAll scanAll = prepareScanAll(namespace1, TABLE_1); selection_SelectionGivenForPreparedWhenCoordinatorStateNotExistAndNotExpired_ShouldNotAbortTransaction( - scanAll, commitType); + scanAll, commitType, isolation, true); } @Test @@ -5317,23 +5546,15 @@ public void getScanner_InReadOnlyMode_WithSerializable_ShouldNotThrowAnyExceptio originalStorage.mutate(prepareMutationComposer.get()); // Act Assert - Optional actual; - while (true) { - try { - actual = - manager.get( - Get.newBuilder() - .namespace(namespace1) - .table(TABLE_1) - .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) - .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) - .where(column(BALANCE).isEqualToInt(INITIAL_BALANCE)) - .build()); - break; - } catch (CrudConflictException e) { - // Retry on conflict - } - } + Optional actual = + manager.get( + Get.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .clusteringKey(Key.ofInt(ACCOUNT_TYPE, 0)) + .where(column(BALANCE).isEqualToInt(INITIAL_BALANCE)) + .build()); assertThat(actual).isPresent(); assertThat(actual.get().getInt(ACCOUNT_ID)).isEqualTo(0); @@ -5454,22 +5675,14 @@ public void getScanner_InReadOnlyMode_WithSerializable_ShouldNotThrowAnyExceptio originalStorage.mutate(prepareMutationComposer.get()); // Act Assert - List results; - while (true) { - try { - results = - manager.scan( - Scan.newBuilder() - .namespace(namespace1) - .table(TABLE_1) - .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) - .where(column(BALANCE).isEqualToInt(INITIAL_BALANCE)) - .build()); - break; - } catch (CrudConflictException e) { - // Retry on conflict - } - } + List results = + manager.scan( + Scan.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .where(column(BALANCE).isEqualToInt(INITIAL_BALANCE)) + .build()); assertThat(results).hasSize(2); assertThat(results.get(0).getInt(ACCOUNT_ID)).isEqualTo(0); @@ -5696,20 +5909,15 @@ public void getScanner_InReadOnlyMode_WithSerializable_ShouldNotThrowAnyExceptio // Act Assert List results; - while (true) { - try (TransactionManagerCrudOperable.Scanner scanner = - manager.getScanner( - Scan.newBuilder() - .namespace(namespace1) - .table(TABLE_1) - .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) - .where(column(BALANCE).isEqualToInt(INITIAL_BALANCE)) - .build())) { - results = scanner.all(); - break; - } catch (CrudConflictException e) { - // Retry on conflict - } + try (TransactionManagerCrudOperable.Scanner scanner = + manager.getScanner( + Scan.newBuilder() + .namespace(namespace1) + .table(TABLE_1) + .partitionKey(Key.ofInt(ACCOUNT_ID, 0)) + .where(column(BALANCE).isEqualToInt(INITIAL_BALANCE)) + .build())) { + results = scanner.all(); } assertThat(results).hasSize(2); diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitSpecificIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitSpecificIntegrationTestBase.java index cbad6a4459..bb416ef610 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitSpecificIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitSpecificIntegrationTestBase.java @@ -260,19 +260,6 @@ private void selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_Sh TwoPhaseCommitTransaction transaction = manager1.begin(); // Act Assert - assertThatThrownBy( - () -> { - if (selectionType == SelectionType.GET) { - transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else if (selectionType == SelectionType.SCAN) { - transaction.scan(prepareScan(0, 0, 0, namespace1, TABLE_1)); - } else { - transaction.scan(prepareScanAll(namespace1, TABLE_1)); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert Optional result; if (selectionType == SelectionType.GET) { result = transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); @@ -285,9 +272,11 @@ private void selection_SelectionGivenForPreparedWhenCoordinatorStateCommitted_Sh assertThat(results.size()).isEqualTo(1); result = Optional.of(results.get(0)); } + transaction.prepare(); transaction.commit(); + // Assert assertThat(result.isPresent()).isTrue(); assertThat(getAccountId(result.get())).isEqualTo(0); assertThat(getAccountType(result.get())).isEqualTo(0); @@ -319,19 +308,6 @@ private void selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_Shou TwoPhaseCommitTransaction transaction = manager1.begin(); // Act Assert - assertThatThrownBy( - () -> { - if (selectionType == SelectionType.GET) { - transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else if (selectionType == SelectionType.SCAN) { - transaction.scan(prepareScan(0, 0, 0, namespace1, TABLE_1)); - } else { - transaction.scan(prepareScanAll(namespace1, TABLE_1)); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert Optional result; if (selectionType == SelectionType.GET) { result = transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); @@ -344,9 +320,11 @@ private void selection_SelectionGivenForPreparedWhenCoordinatorStateAborted_Shou assertThat(results.size()).isEqualTo(1); result = Optional.of(results.get(0)); } + transaction.prepare(); transaction.commit(); + // Assert assertThat(result.isPresent()).isTrue(); assertThat(getAccountId(result.get())).isEqualTo(0); assertThat(getAccountType(result.get())).isEqualTo(0); @@ -425,19 +403,6 @@ public void scan_ScanGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( TwoPhaseCommitTransaction transaction = manager1.begin(); // Act Assert - assertThatThrownBy( - () -> { - if (selectionType == SelectionType.GET) { - transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else if (selectionType == SelectionType.SCAN) { - transaction.scan(prepareScan(0, 0, 0, namespace1, TABLE_1)); - } else { - transaction.scan(prepareScanAll(namespace1, TABLE_1)); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert Optional result; if (selectionType == SelectionType.GET) { result = transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); @@ -453,11 +418,15 @@ public void scan_ScanGivenForPreparedWhenCoordinatorStateAborted_ShouldRollback( transaction.prepare(); transaction.commit(); + // Assert assertThat(result.isPresent()).isTrue(); assertThat(getAccountId(result.get())).isEqualTo(0); assertThat(getAccountType(result.get())).isEqualTo(0); assertThat(getBalance(result.get())).isEqualTo(0); // a rolled back value + // Wait for the recovery to complete + ((TwoPhaseConsensusCommit) transaction).getCrudHandler().waitForRecoveryCompletion(); + assertThat(coordinatorForStorage1.getState(ANY_ID_2).isPresent()).isTrue(); assertThat(coordinatorForStorage1.getState(ANY_ID_2).get().getState()) .isEqualTo(TransactionState.ABORTED); @@ -478,160 +447,6 @@ public void get_GetGivenForPreparedWhenCoordinatorStateNotExistAndExpired_Should SelectionType.SCAN); } - private void - selection_SelectionGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - SelectionType selectionType) - throws ExecutionException, TransactionException, CoordinatorException { - // Arrange - long current = System.currentTimeMillis(); - populatePreparedRecordAndCoordinatorStateRecordForStorage1( - TransactionState.PREPARED, current, TransactionState.COMMITTED); - - TwoPhaseConsensusCommit transaction = (TwoPhaseConsensusCommit) manager1.begin(); - - transaction.setBeforeRecoveryHook( - () -> - assertThatThrownBy( - () -> { - TwoPhaseCommitTransaction another = manager1.begin(); - if (selectionType == SelectionType.GET) { - another.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else if (selectionType == SelectionType.SCAN) { - another.scan(prepareScan(0, 0, 0, namespace1, TABLE_1)); - } else { - another.scan(prepareScanAll(namespace1, TABLE_1)); - } - }) - .isInstanceOf(UncommittedRecordException.class)); - - // Act Assert - assertThatThrownBy( - () -> { - if (selectionType == SelectionType.GET) { - transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else if (selectionType == SelectionType.SCAN) { - transaction.scan(prepareScan(0, 0, 0, namespace1, TABLE_1)); - } else { - transaction.scan(prepareScanAll(namespace1, TABLE_1)); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - Optional result; - if (selectionType == SelectionType.GET) { - result = transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else { - Scan scan = - selectionType == SelectionType.SCAN - ? prepareScan(0, 0, 0, namespace1, TABLE_1) - : prepareScanAll(namespace1, TABLE_1); - List results = transaction.scan(scan); - assertThat(results.size()).isEqualTo(1); - result = Optional.of(results.get(0)); - } - transaction.prepare(); - transaction.commit(); - - assertThat(result.isPresent()).isTrue(); - assertThat(getAccountId(result.get())).isEqualTo(0); - assertThat(getAccountType(result.get())).isEqualTo(0); - assertThat(getBalance(result.get())).isEqualTo(INITIAL_BALANCE); // a rolled forward value - } - - @Test - public void - get_GetGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly() - throws ExecutionException, TransactionException, CoordinatorException { - selection_SelectionGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - SelectionType.GET); - } - - @Test - public void - scan_ScanGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly() - throws ExecutionException, TransactionException, CoordinatorException { - selection_SelectionGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - SelectionType.SCAN); - } - - private void - selection_SelectionGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - SelectionType selectionType) - throws ExecutionException, TransactionException, CoordinatorException { - // Arrange - long current = System.currentTimeMillis(); - populatePreparedRecordAndCoordinatorStateRecordForStorage1( - TransactionState.PREPARED, current, TransactionState.ABORTED); - - TwoPhaseConsensusCommit transaction = (TwoPhaseConsensusCommit) manager1.begin(); - - transaction.setBeforeRecoveryHook( - () -> - assertThatThrownBy( - () -> { - TwoPhaseCommitTransaction another = manager1.begin(); - if (selectionType == SelectionType.GET) { - another.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else if (selectionType == SelectionType.SCAN) { - another.scan(prepareScan(0, 0, 0, namespace1, TABLE_1)); - } else { - another.scan(prepareScanAll(namespace1, TABLE_1)); - } - }) - .isInstanceOf(UncommittedRecordException.class)); - - // Act Assert - assertThatThrownBy( - () -> { - if (selectionType == SelectionType.GET) { - transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else if (selectionType == SelectionType.SCAN) { - transaction.scan(prepareScan(0, 0, 0, namespace1, TABLE_1)); - } else { - transaction.scan(prepareScanAll(namespace1, TABLE_1)); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - Optional result; - if (selectionType == SelectionType.GET) { - result = transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else { - Scan scan = - selectionType == SelectionType.SCAN - ? prepareScan(0, 0, 0, namespace1, TABLE_1) - : prepareScanAll(namespace1, TABLE_1); - List results = transaction.scan(scan); - assertThat(results.size()).isEqualTo(1); - result = Optional.of(results.get(0)); - } - transaction.prepare(); - transaction.commit(); - - assertThat(result.isPresent()).isTrue(); - assertThat(getAccountId(result.get())).isEqualTo(0); - assertThat(getAccountType(result.get())).isEqualTo(0); - assertThat(getBalance(result.get())).isEqualTo(0); // a rolled back value - } - - @Test - public void - get_GetGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly() - throws ExecutionException, TransactionException, CoordinatorException { - selection_SelectionGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - SelectionType.GET); - } - - @Test - public void - scan_ScanGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly() - throws ExecutionException, TransactionException, CoordinatorException { - selection_SelectionGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - SelectionType.SCAN); - } - private void selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward( SelectionType selectionType) throws ExecutionException, TransactionException, CoordinatorException { @@ -643,19 +458,6 @@ private void selection_SelectionGivenForDeletedWhenCoordinatorStateCommitted_Sho TwoPhaseCommitTransaction transaction = manager1.begin(); // Act Assert - assertThatThrownBy( - () -> { - if (selectionType == SelectionType.GET) { - transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else if (selectionType == SelectionType.SCAN) { - transaction.scan(prepareScan(0, 0, 0, namespace1, TABLE_1)); - } else { - transaction.scan(prepareScanAll(namespace1, TABLE_1)); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert if (selectionType == SelectionType.GET) { Optional result = transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); assertThat(result.isPresent()).isFalse(); // deleted @@ -696,19 +498,6 @@ private void selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_Shoul TwoPhaseCommitTransaction transaction = manager1.begin(); // Act Assert - assertThatThrownBy( - () -> { - if (selectionType == SelectionType.GET) { - transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else if (selectionType == SelectionType.SCAN) { - transaction.scan(prepareScan(0, 0, 0, namespace1, TABLE_1)); - } else { - transaction.scan(prepareScanAll(namespace1, TABLE_1)); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert Optional result; if (selectionType == SelectionType.GET) { result = transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); @@ -724,6 +513,7 @@ private void selection_SelectionGivenForDeletedWhenCoordinatorStateAborted_Shoul transaction.prepare(); transaction.commit(); + // Assert assertThat(result.isPresent()).isTrue(); assertThat(getAccountId(result.get())).isEqualTo(0); assertThat(getAccountType(result.get())).isEqualTo(0); @@ -801,19 +591,6 @@ public void scan_ScanGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback() TwoPhaseCommitTransaction transaction = manager1.begin(); // Act Assert - assertThatThrownBy( - () -> { - if (selectionType == SelectionType.GET) { - transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else if (selectionType == SelectionType.SCAN) { - transaction.scan(prepareScan(0, 0, 0, namespace1, TABLE_1)); - } else { - transaction.scan(prepareScanAll(namespace1, TABLE_1)); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert Optional result; if (selectionType == SelectionType.GET) { result = transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); @@ -829,11 +606,15 @@ public void scan_ScanGivenForDeletedWhenCoordinatorStateAborted_ShouldRollback() transaction.prepare(); transaction.commit(); + // Assert assertThat(result.isPresent()).isTrue(); assertThat(getAccountId(result.get())).isEqualTo(0); assertThat(getAccountType(result.get())).isEqualTo(0); assertThat(getBalance(result.get())).isEqualTo(0); // a rolled back value + // Wait for the recovery to complete + ((TwoPhaseConsensusCommit) transaction).getCrudHandler().waitForRecoveryCompletion(); + assertThat(coordinatorForStorage1.getState(ANY_ID_2).isPresent()).isTrue(); assertThat(coordinatorForStorage1.getState(ANY_ID_2).get().getState()) .isEqualTo(TransactionState.ABORTED); @@ -854,154 +635,6 @@ public void get_GetGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldA SelectionType.SCAN); } - private void - selection_SelectionGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - SelectionType selectionType) - throws ExecutionException, TransactionException, CoordinatorException { - // Arrange - long current = System.currentTimeMillis(); - populatePreparedRecordAndCoordinatorStateRecordForStorage1( - TransactionState.DELETED, current, TransactionState.COMMITTED); - - TwoPhaseConsensusCommit transaction = (TwoPhaseConsensusCommit) manager1.begin(); - - transaction.setBeforeRecoveryHook( - () -> - assertThatThrownBy( - () -> { - TwoPhaseCommitTransaction another = manager1.begin(); - if (selectionType == SelectionType.GET) { - another.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else if (selectionType == SelectionType.SCAN) { - another.scan(prepareScan(0, 0, 0, namespace1, TABLE_1)); - } else { - another.scan(prepareScanAll(namespace1, TABLE_1)); - } - }) - .isInstanceOf(UncommittedRecordException.class)); - - // Act Assert - assertThatThrownBy( - () -> { - if (selectionType == SelectionType.GET) { - transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else if (selectionType == SelectionType.SCAN) { - transaction.scan(prepareScan(0, 0, 0, namespace1, TABLE_1)); - } else { - transaction.scan(prepareScanAll(namespace1, TABLE_1)); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - if (selectionType == SelectionType.GET) { - Optional result = transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); - assertThat(result.isPresent()).isFalse(); // deleted - } else { - Scan scan = - selectionType == SelectionType.SCAN - ? prepareScan(0, 0, 0, namespace1, TABLE_1) - : prepareScanAll(namespace1, TABLE_1); - List results = transaction.scan(scan); - assertThat(results.size()).isEqualTo(0); // deleted - } - transaction.prepare(); - transaction.commit(); - } - - @Test - public void - get_GetGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly() - throws ExecutionException, TransactionException, CoordinatorException { - selection_SelectionGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - SelectionType.GET); - } - - @Test - public void - scan_ScanGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly() - throws ExecutionException, TransactionException, CoordinatorException { - selection_SelectionGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - SelectionType.SCAN); - } - - private void - selection_SelectionGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - SelectionType selectionType) - throws ExecutionException, TransactionException, CoordinatorException { - // Arrange - long current = System.currentTimeMillis(); - populatePreparedRecordAndCoordinatorStateRecordForStorage1( - TransactionState.DELETED, current, TransactionState.ABORTED); - - TwoPhaseConsensusCommit transaction = (TwoPhaseConsensusCommit) manager1.begin(); - - transaction.setBeforeRecoveryHook( - () -> - assertThatThrownBy( - () -> { - TwoPhaseCommitTransaction another = manager1.begin(); - if (selectionType == SelectionType.GET) { - another.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else if (selectionType == SelectionType.SCAN) { - another.scan(prepareScan(0, 0, 0, namespace1, TABLE_1)); - } else { - another.scan(prepareScanAll(namespace1, TABLE_1)); - } - }) - .isInstanceOf(UncommittedRecordException.class)); - - // Act Assert - assertThatThrownBy( - () -> { - if (selectionType == SelectionType.GET) { - transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else if (selectionType == SelectionType.SCAN) { - transaction.scan(prepareScan(0, 0, 0, namespace1, TABLE_1)); - } else { - transaction.scan(prepareScanAll(namespace1, TABLE_1)); - } - }) - .isInstanceOf(UncommittedRecordException.class); - - // Assert - Optional result; - if (selectionType == SelectionType.GET) { - result = transaction.get(prepareGet(0, 0, namespace1, TABLE_1)); - } else { - Scan scan = - selectionType == SelectionType.SCAN - ? prepareScan(0, 0, 0, namespace1, TABLE_1) - : prepareScanAll(namespace1, TABLE_1); - List results = transaction.scan(scan); - assertThat(results.size()).isEqualTo(1); - result = Optional.of(results.get(0)); - } - transaction.prepare(); - transaction.commit(); - - assertThat(result.isPresent()).isTrue(); - assertThat(getAccountId(result.get())).isEqualTo(0); - assertThat(getAccountType(result.get())).isEqualTo(0); - assertThat(getBalance(result.get())).isEqualTo(0); // a rolled back value - } - - @Test - public void - get_GetGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly() - throws ExecutionException, TransactionException, CoordinatorException { - selection_SelectionGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - SelectionType.GET); - } - - @Test - public void - scan_ScanGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly() - throws ExecutionException, TransactionException, CoordinatorException { - selection_SelectionGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - SelectionType.SCAN); - } - @Test public void getThenScanAndGet_CommitHappenedInBetween_OnlyGetShouldReadRepeatably() throws TransactionException { @@ -2625,14 +2258,6 @@ public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateAborted_ShouldRoll SelectionType.SCAN_ALL); } - @Test - public void - scanAll_ScanAllGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly() - throws ExecutionException, CoordinatorException, TransactionException { - selection_SelectionGivenForDeletedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - SelectionType.SCAN_ALL); - } - @Test public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateCommitted_ShouldRollforward() throws ExecutionException, CoordinatorException, TransactionException { @@ -2640,14 +2265,6 @@ public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateCommitted_ShouldRo SelectionType.SCAN_ALL); } - @Test - public void - scanAll_ScanAllGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly() - throws ExecutionException, CoordinatorException, TransactionException { - selection_SelectionGivenForDeletedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - SelectionType.SCAN_ALL); - } - @Test public void scanAll_ScanAllGivenForDeletedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() @@ -2695,14 +2312,6 @@ public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateAborted_ShouldRol SelectionType.SCAN_ALL); } - @Test - public void - scanAll_ScanAllGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly() - throws ExecutionException, CoordinatorException, TransactionException { - selection_SelectionGivenForPreparedWhenCoordinatorStateAbortedAndRolledBackByAnother_ShouldRollbackProperly( - SelectionType.SCAN_ALL); - } - @Test public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateCommitted_ShouldRollforward() throws ExecutionException, CoordinatorException, TransactionException { @@ -2710,14 +2319,6 @@ public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateCommitted_ShouldR SelectionType.SCAN_ALL); } - @Test - public void - scanAll_ScanAllGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly() - throws ExecutionException, CoordinatorException, TransactionException { - selection_SelectionGivenForPreparedWhenCoordinatorStateCommittedAndRolledForwardByAnother_ShouldRollforwardProperly( - SelectionType.SCAN_ALL); - } - @Test public void scanAll_ScanAllGivenForPreparedWhenCoordinatorStateNotExistAndExpired_ShouldAbortTransaction() From b55e8904a531a44ddd30b5910e4b4f4d00aeab75 Mon Sep 17 00:00:00 2001 From: brfrn169 Date: Thu, 19 Jun 2025 09:48:52 +0900 Subject: [PATCH 2/6] Fix based on feedback --- .../consensuscommit/RollbackMutationComposer.java | 3 ++- .../test/java/com/scalar/db/io/DateColumnTest.java | 2 +- .../db/storage/jdbc/RdbEngineOracleTest.java | 1 - .../TwoPhaseConsensusCommitTest.java | 2 +- .../db/util/groupcommit/GroupCommitterTest.java | 2 +- .../db/util/groupcommit/GroupManagerTest.java | 2 +- .../core/dataimport/dao/ScalarDbDaoTest.java | 14 ++++++++++++-- 7 files changed, 18 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposer.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposer.java index 25d9166b78..b2fee4b643 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposer.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposer.java @@ -5,7 +5,8 @@ import static com.scalar.db.transaction.consensuscommit.Attribute.STATE; import static com.scalar.db.transaction.consensuscommit.Attribute.toIdValue; import static com.scalar.db.transaction.consensuscommit.Attribute.toStateValue; -import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.*; +import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.extractAfterImageColumnsFromBeforeImage; +import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.getTransactionTableMetadata; import com.scalar.db.api.ConditionBuilder; import com.scalar.db.api.ConditionalExpression; diff --git a/core/src/test/java/com/scalar/db/io/DateColumnTest.java b/core/src/test/java/com/scalar/db/io/DateColumnTest.java index f8465607f1..7c8bd9a9ff 100644 --- a/core/src/test/java/com/scalar/db/io/DateColumnTest.java +++ b/core/src/test/java/com/scalar/db/io/DateColumnTest.java @@ -2,7 +2,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import java.time.Clock; import java.time.LocalDate; diff --git a/core/src/test/java/com/scalar/db/storage/jdbc/RdbEngineOracleTest.java b/core/src/test/java/com/scalar/db/storage/jdbc/RdbEngineOracleTest.java index 22115440a3..88e64cb8f9 100644 --- a/core/src/test/java/com/scalar/db/storage/jdbc/RdbEngineOracleTest.java +++ b/core/src/test/java/com/scalar/db/storage/jdbc/RdbEngineOracleTest.java @@ -1,7 +1,6 @@ package com.scalar.db.storage.jdbc; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.*; import com.scalar.db.api.Scan.Ordering.Order; import com.scalar.db.api.TableMetadata; diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitTest.java index 36ef01e5c6..6ef7fc4979 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitTest.java @@ -66,7 +66,7 @@ public class TwoPhaseConsensusCommitTest { public void setUp() throws Exception { MockitoAnnotations.openMocks(this).close(); - // Arrange1 + // Arrange transaction = new TwoPhaseConsensusCommit(crud, commit, mutationOperationChecker); when(crud.areAllScannersClosed()).thenReturn(true); diff --git a/core/src/test/java/com/scalar/db/util/groupcommit/GroupCommitterTest.java b/core/src/test/java/com/scalar/db/util/groupcommit/GroupCommitterTest.java index d303ba14a3..d4f13a47ef 100644 --- a/core/src/test/java/com/scalar/db/util/groupcommit/GroupCommitterTest.java +++ b/core/src/test/java/com/scalar/db/util/groupcommit/GroupCommitterTest.java @@ -1,7 +1,7 @@ package com.scalar.db.util.groupcommit; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.never; diff --git a/core/src/test/java/com/scalar/db/util/groupcommit/GroupManagerTest.java b/core/src/test/java/com/scalar/db/util/groupcommit/GroupManagerTest.java index 2816570536..7193a8d730 100644 --- a/core/src/test/java/com/scalar/db/util/groupcommit/GroupManagerTest.java +++ b/core/src/test/java/com/scalar/db/util/groupcommit/GroupManagerTest.java @@ -1,7 +1,7 @@ package com.scalar.db.util.groupcommit; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.util.concurrent.Uninterruptibles; import com.scalar.db.util.groupcommit.GroupCommitKeyManipulator.Keys; diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java index cc1798e2f8..58a62203e8 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/dao/ScalarDbDaoTest.java @@ -1,13 +1,23 @@ package com.scalar.db.dataloader.core.dataimport.dao; -import static com.scalar.db.dataloader.core.UnitTestUtils.*; +import static com.scalar.db.dataloader.core.UnitTestUtils.TEST_COLUMN_1_PK; +import static com.scalar.db.dataloader.core.UnitTestUtils.TEST_COLUMN_2_CK; +import static com.scalar.db.dataloader.core.UnitTestUtils.TEST_COLUMN_4; +import static com.scalar.db.dataloader.core.UnitTestUtils.TEST_COLUMN_5; +import static com.scalar.db.dataloader.core.UnitTestUtils.TEST_COLUMN_6; +import static com.scalar.db.dataloader.core.UnitTestUtils.TEST_NAMESPACE; +import static com.scalar.db.dataloader.core.UnitTestUtils.TEST_TABLE_NAME; +import static com.scalar.db.dataloader.core.UnitTestUtils.TEST_VALUE_INT; +import static com.scalar.db.dataloader.core.UnitTestUtils.TEST_VALUE_LONG; import static org.assertj.core.api.Assertions.assertThat; import com.scalar.db.api.Scan; import com.scalar.db.api.ScanBuilder; import com.scalar.db.dataloader.core.ScanRange; import com.scalar.db.io.Key; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; From 309e61fdfff36dca18f78e0b4f73386d6e54dcf3 Mon Sep 17 00:00:00 2001 From: brfrn169 Date: Thu, 19 Jun 2025 13:34:38 +0900 Subject: [PATCH 3/6] [skip ci] Rename --- .../consensuscommit/ConsensusCommitUtils.java | 2 +- .../consensuscommit/RecoveryExecutor.java | 12 ++++++------ .../consensuscommit/RollbackMutationComposer.java | 4 ++-- .../consensuscommit/ConsensusCommitUtilsTest.java | 12 ++++++------ 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtils.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtils.java index d156a38a13..d4f5d18d7a 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtils.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtils.java @@ -319,7 +319,7 @@ public static int getNextTxVersion(@Nullable Integer currentTxVersion) { } } - static void extractAfterImageColumnsFromBeforeImage( + static void createAfterImageColumnsFromBeforeImage( Map> columns, TransactionResult result, Set beforeImageColumnNames) { diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutor.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutor.java index b860130612..b8b776767b 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutor.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutor.java @@ -1,6 +1,6 @@ package com.scalar.db.transaction.consensuscommit; -import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.extractAfterImageColumnsFromBeforeImage; +import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.createAfterImageColumnsFromBeforeImage; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -102,10 +102,10 @@ private Optional createRecoveredResult( throwUncommittedRecordExceptionIfTransactionNotExpired(state, selection, result, transactionId); if (!state.isPresent() || state.get().getState() == TransactionState.ABORTED) { - return createRolledBackRecord(selection, result, transactionId); + return createRecordFromBeforeImage(selection, result, transactionId); } else { assert state.get().getState() == TransactionState.COMMITTED; - return createRolledForwardResult(selection, result, transactionId); + return createResultFromAfterImage(selection, result, transactionId); } } @@ -124,7 +124,7 @@ private void throwUncommittedRecordExceptionIfTransactionNotExpired( } } - private Optional createRolledBackRecord( + private Optional createRecordFromBeforeImage( Selection selection, TransactionResult result, String transactionId) throws CrudException { if (!result.hasBeforeImage()) { return Optional.empty(); @@ -138,7 +138,7 @@ private Optional createRolledBackRecord( Map> columns = new HashMap<>(); - extractAfterImageColumnsFromBeforeImage(columns, result, beforeImageColumnNames); + createAfterImageColumnsFromBeforeImage(columns, result, beforeImageColumnNames); Key partitionKey = ScalarDbUtils.getPartitionKey(result, tableMetadata); partitionKey.getColumns().forEach(c -> columns.put(c.getName(), c)); @@ -151,7 +151,7 @@ private Optional createRolledBackRecord( return Optional.of(new TransactionResult(new ResultImpl(columns, tableMetadata))); } - private Optional createRolledForwardResult( + private Optional createResultFromAfterImage( Selection selection, TransactionResult result, String transactionId) throws CrudException { if (result.getState() == TransactionState.DELETED) { return Optional.empty(); diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposer.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposer.java index b2fee4b643..8669bd4193 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposer.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/RollbackMutationComposer.java @@ -5,7 +5,7 @@ import static com.scalar.db.transaction.consensuscommit.Attribute.STATE; import static com.scalar.db.transaction.consensuscommit.Attribute.toIdValue; import static com.scalar.db.transaction.consensuscommit.Attribute.toStateValue; -import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.extractAfterImageColumnsFromBeforeImage; +import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.createAfterImageColumnsFromBeforeImage; import static com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils.getTransactionTableMetadata; import com.scalar.db.api.ConditionBuilder; @@ -106,7 +106,7 @@ private Put composePut(Operation base, TransactionResult result) throws Executio clusteringKey.ifPresent(putBuilder::clusteringKey); Map> columns = new HashMap<>(); - extractAfterImageColumnsFromBeforeImage(columns, result, beforeImageColumnNames); + createAfterImageColumnsFromBeforeImage(columns, result, beforeImageColumnNames); columns.values().forEach(putBuilder::value); // Set before image columns to null diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtilsTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtilsTest.java index 1eb8b6565f..5ccd98ea53 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtilsTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtilsTest.java @@ -625,7 +625,7 @@ void getNextTxVersion_LargeValueGiven_shouldReturnNextVersion() { } @Test - public void extractAfterImageColumnsFromBeforeImage_shouldExtractCorrectly() { + public void createAfterImageColumnsFromBeforeImage_shouldExtractCorrectly() { // Arrange Map> columns = new HashMap<>(); Set beforeImageColumnNames = new HashSet<>(); @@ -641,7 +641,7 @@ public void extractAfterImageColumnsFromBeforeImage_shouldExtractCorrectly() { when(result.getColumns()).thenReturn(resultColumns); // Act - ConsensusCommitUtils.extractAfterImageColumnsFromBeforeImage( + ConsensusCommitUtils.createAfterImageColumnsFromBeforeImage( columns, result, beforeImageColumnNames); // Assert @@ -655,7 +655,7 @@ public void extractAfterImageColumnsFromBeforeImage_shouldExtractCorrectly() { @Test public void - extractAfterImageColumnsFromBeforeImage_versionColumnWithZero_shouldCreateNullVersion() { + createAfterImageColumnsFromBeforeImage_versionColumnWithZero_shouldCreateNullVersion() { // Arrange Map> columns = new HashMap<>(); Set beforeImageColumnNames = new HashSet<>(); @@ -668,7 +668,7 @@ public void extractAfterImageColumnsFromBeforeImage_shouldExtractCorrectly() { when(result.getColumns()).thenReturn(resultColumns); // Act - ConsensusCommitUtils.extractAfterImageColumnsFromBeforeImage( + ConsensusCommitUtils.createAfterImageColumnsFromBeforeImage( columns, result, beforeImageColumnNames); // Assert @@ -678,7 +678,7 @@ public void extractAfterImageColumnsFromBeforeImage_shouldExtractCorrectly() { } @Test - public void extractAfterImageColumnsFromBeforeImage_versionColumnWithNonZero_shouldCopyValue() { + public void createAfterImageColumnsFromBeforeImage_versionColumnWithNonZero_shouldCopyValue() { // Arrange Map> columns = new HashMap<>(); Set beforeImageColumnNames = new HashSet<>(); @@ -691,7 +691,7 @@ public void extractAfterImageColumnsFromBeforeImage_versionColumnWithNonZero_sho when(result.getColumns()).thenReturn(resultColumns); // Act - ConsensusCommitUtils.extractAfterImageColumnsFromBeforeImage( + ConsensusCommitUtils.createAfterImageColumnsFromBeforeImage( columns, result, beforeImageColumnNames); // Assert From 24c4822e3e3b4347ea83571a70ea56d46fb0fcdc Mon Sep 17 00:00:00 2001 From: brfrn169 Date: Thu, 19 Jun 2025 17:28:23 +0900 Subject: [PATCH 4/6] Use CachedThreadPool for RecoveryExecutor --- .../consensuscommit/ConsensusCommitConfig.java | 10 ---------- .../consensuscommit/ConsensusCommitManager.java | 8 ++------ .../consensuscommit/RecoveryExecutor.java | 6 ++---- .../TwoPhaseConsensusCommitManager.java | 8 ++------ .../consensuscommit/ConsensusCommitConfigTest.java | 14 -------------- .../consensuscommit/RecoveryExecutorTest.java | 2 +- ...ensusCommitNullMetadataIntegrationTestBase.java | 7 +------ ...ConsensusCommitSpecificIntegrationTestBase.java | 7 +------ 8 files changed, 9 insertions(+), 53 deletions(-) diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfig.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfig.java index cf7e1fea60..7e1b9c5c07 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfig.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfig.java @@ -35,7 +35,6 @@ public class ConsensusCommitConfig { public static final String COORDINATOR_WRITE_OMISSION_ON_READ_ONLY_ENABLED = PREFIX + "coordinator.write_omission_on_read_only.enabled"; - public static final String RECOVERY_EXECUTOR_COUNT = PREFIX + "recovery_executor_count"; public static final String PARALLEL_IMPLICIT_PRE_READ = PREFIX + "parallel_implicit_pre_read.enabled"; public static final String INCLUDE_METADATA_ENABLED = PREFIX + "include_metadata.enabled"; @@ -57,7 +56,6 @@ public class ConsensusCommitConfig { COORDINATOR_GROUP_COMMIT_PREFIX + "metrics_monitor_log_enabled"; public static final int DEFAULT_PARALLEL_EXECUTOR_COUNT = 128; - public static final int DEFAULT_RECOVERY_EXECUTOR_COUNT = 128; public static final int DEFAULT_COORDINATOR_GROUP_COMMIT_SLOT_CAPACITY = 20; public static final int DEFAULT_COORDINATOR_GROUP_COMMIT_GROUP_SIZE_FIX_TIMEOUT_MILLIS = 40; @@ -77,7 +75,6 @@ public class ConsensusCommitConfig { private final boolean asyncRollbackEnabled; private final boolean coordinatorWriteOmissionOnReadOnlyEnabled; - private final int recoveryExecutorCount; private final boolean parallelImplicitPreReadEnabled; private final boolean isIncludeMetadataEnabled; @@ -148,9 +145,6 @@ public ConsensusCommitConfig(DatabaseConfig databaseConfig) { coordinatorWriteOmissionOnReadOnlyEnabled = getBoolean(properties, COORDINATOR_WRITE_OMISSION_ON_READ_ONLY_ENABLED, true); - recoveryExecutorCount = - getInt(properties, RECOVERY_EXECUTOR_COUNT, DEFAULT_RECOVERY_EXECUTOR_COUNT); - isIncludeMetadataEnabled = getBoolean(properties, INCLUDE_METADATA_ENABLED, false); parallelImplicitPreReadEnabled = getBoolean(properties, PARALLEL_IMPLICIT_PRE_READ, true); @@ -225,10 +219,6 @@ public boolean isCoordinatorWriteOmissionOnReadOnlyEnabled() { return coordinatorWriteOmissionOnReadOnlyEnabled; } - public int getRecoveryExecutorCount() { - return recoveryExecutorCount; - } - public boolean isParallelImplicitPreReadEnabled() { return parallelImplicitPreReadEnabled; } diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java index b18ddf8812..442975841f 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitManager.java @@ -72,9 +72,7 @@ public ConsensusCommitManager( new TransactionTableMetadataManager( admin, databaseConfig.getMetadataCacheExpirationTimeSecs()); RecoveryHandler recovery = new RecoveryHandler(storage, coordinator, tableMetadataManager); - recoveryExecutor = - new RecoveryExecutor( - coordinator, recovery, tableMetadataManager, config.getRecoveryExecutorCount()); + recoveryExecutor = new RecoveryExecutor(coordinator, recovery, tableMetadataManager); groupCommitter = CoordinatorGroupCommitter.from(config).orElse(null); commit = createCommitHandler(); isIncludeMetadataEnabled = config.isIncludeMetadataEnabled(); @@ -94,9 +92,7 @@ protected ConsensusCommitManager(DatabaseConfig databaseConfig) { new TransactionTableMetadataManager( admin, databaseConfig.getMetadataCacheExpirationTimeSecs()); RecoveryHandler recovery = new RecoveryHandler(storage, coordinator, tableMetadataManager); - recoveryExecutor = - new RecoveryExecutor( - coordinator, recovery, tableMetadataManager, config.getRecoveryExecutorCount()); + recoveryExecutor = new RecoveryExecutor(coordinator, recovery, tableMetadataManager); groupCommitter = CoordinatorGroupCommitter.from(config).orElse(null); commit = createCommitHandler(); isIncludeMetadataEnabled = config.isIncludeMetadataEnabled(); diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutor.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutor.java index b8b776767b..20372718b3 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutor.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutor.java @@ -49,14 +49,12 @@ public class RecoveryExecutor implements AutoCloseable { public RecoveryExecutor( Coordinator coordinator, RecoveryHandler recovery, - TransactionTableMetadataManager tableMetadataManager, - int threadPoolSize) { + TransactionTableMetadataManager tableMetadataManager) { this.coordinator = Objects.requireNonNull(coordinator); this.recovery = Objects.requireNonNull(recovery); this.tableMetadataManager = Objects.requireNonNull(tableMetadataManager); executorService = - Executors.newFixedThreadPool( - threadPoolSize, + Executors.newCachedThreadPool( new ThreadFactoryBuilder() .setNameFormat("recovery-executor-%d") .setDaemon(true) diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java index f8e4e30793..cb38e72356 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/TwoPhaseConsensusCommitManager.java @@ -74,9 +74,7 @@ public TwoPhaseConsensusCommitManager( coordinator = new Coordinator(storage, config); parallelExecutor = new ParallelExecutor(config); RecoveryHandler recovery = new RecoveryHandler(storage, coordinator, tableMetadataManager); - recoveryExecutor = - new RecoveryExecutor( - coordinator, recovery, tableMetadataManager, config.getRecoveryExecutorCount()); + recoveryExecutor = new RecoveryExecutor(coordinator, recovery, tableMetadataManager); commit = new CommitHandler( storage, @@ -100,9 +98,7 @@ public TwoPhaseConsensusCommitManager(DatabaseConfig databaseConfig) { coordinator = new Coordinator(storage, config); parallelExecutor = new ParallelExecutor(config); RecoveryHandler recovery = new RecoveryHandler(storage, coordinator, tableMetadataManager); - recoveryExecutor = - new RecoveryExecutor( - coordinator, recovery, tableMetadataManager, config.getRecoveryExecutorCount()); + recoveryExecutor = new RecoveryExecutor(coordinator, recovery, tableMetadataManager); commit = new CommitHandler( storage, diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfigTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfigTest.java index c9cb63eaf9..d6ad3811c5 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfigTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfigTest.java @@ -28,7 +28,6 @@ public void constructor_NoPropertiesGiven_ShouldLoadAsDefaultValues() { assertThat(config.isAsyncCommitEnabled()).isFalse(); assertThat(config.isAsyncRollbackEnabled()).isFalse(); assertThat(config.isCoordinatorWriteOmissionOnReadOnlyEnabled()).isTrue(); - assertThat(config.getRecoveryExecutorCount()).isEqualTo(128); assertThat(config.isParallelImplicitPreReadEnabled()).isTrue(); assertThat(config.isIncludeMetadataEnabled()).isFalse(); } @@ -160,19 +159,6 @@ public void constructor_AsyncExecutionRelatedPropertiesGiven_ShouldLoadProperly( constructor_PropertiesWithCoordinatorWriteOmissionOnReadOnlyEnabledGiven_ShouldLoadProperly() { // Arrange Properties props = new Properties(); - props.setProperty(ConsensusCommitConfig.RECOVERY_EXECUTOR_COUNT, "256"); - - // Act - ConsensusCommitConfig config = new ConsensusCommitConfig(new DatabaseConfig(props)); - - // Assert - assertThat(config.getRecoveryExecutorCount()).isEqualTo(256); - } - - @Test - public void constructor_PropertiesWithRecoveryExecutorCountGiven_ShouldLoadProperly() { - // Arrange - Properties props = new Properties(); props.setProperty( ConsensusCommitConfig.COORDINATOR_WRITE_OMISSION_ON_READ_ONLY_ENABLED, "false"); diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutorTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutorTest.java index 45ec9aaff4..db14fe7b3e 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutorTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/RecoveryExecutorTest.java @@ -134,7 +134,7 @@ public class RecoveryExecutorTest { public void setUp() throws Exception { MockitoAnnotations.openMocks(this).close(); - executor = new RecoveryExecutor(coordinator, recovery, tableMetadataManager, 1); + executor = new RecoveryExecutor(coordinator, recovery, tableMetadataManager); // Arrange when(tableMetadataManager.getTransactionTableMetadata(selection)) diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitNullMetadataIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitNullMetadataIntegrationTestBase.java index 60ebed7dc0..7c8301ae9d 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitNullMetadataIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitNullMetadataIntegrationTestBase.java @@ -138,12 +138,7 @@ public void setUp() throws Exception { TransactionTableMetadataManager tableMetadataManager = new TransactionTableMetadataManager(admin, -1); recovery = spy(new RecoveryHandler(storage, coordinator, tableMetadataManager)); - recoveryExecutor = - new RecoveryExecutor( - coordinator, - recovery, - tableMetadataManager, - consensusCommitConfig.getRecoveryExecutorCount()); + recoveryExecutor = new RecoveryExecutor(coordinator, recovery, tableMetadataManager); groupCommitter = CoordinatorGroupCommitter.from(consensusCommitConfig).orElse(null); CommitHandler commit = spy(createCommitHandler(tableMetadataManager, groupCommitter)); manager = diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java index cfff4e913c..3dad47a72d 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java @@ -179,12 +179,7 @@ public void setUp() throws Exception { TransactionTableMetadataManager tableMetadataManager = new TransactionTableMetadataManager(admin, -1); recovery = spy(new RecoveryHandler(storage, coordinator, tableMetadataManager)); - recoveryExecutor = - new RecoveryExecutor( - coordinator, - recovery, - tableMetadataManager, - consensusCommitConfig.getRecoveryExecutorCount()); + recoveryExecutor = new RecoveryExecutor(coordinator, recovery, tableMetadataManager); groupCommitter = CoordinatorGroupCommitter.from(consensusCommitConfig).orElse(null); commit = spy(createCommitHandler(tableMetadataManager, groupCommitter)); manager = From 4271072bb73dd44a3a5b89901b6d28244e45a126 Mon Sep 17 00:00:00 2001 From: brfrn169 Date: Thu, 19 Jun 2025 17:49:30 +0900 Subject: [PATCH 5/6] Fix --- .../scalar/db/transaction/consensuscommit/CrudHandlerTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/CrudHandlerTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/CrudHandlerTest.java index d011531623..cd2816ab2c 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/CrudHandlerTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/CrudHandlerTest.java @@ -568,7 +568,6 @@ public void get_DifferentGetButSameRecordReturned_ShouldNotOverwriteReadSet() Get getForStorage2 = Get.newBuilder(get2) .clearProjections() - .projections(TRANSACTION_TABLE_METADATA.getAfterImageColumnNames()) .clearConditions() .where(column(ANY_NAME_3).isEqualToText(ANY_TEXT_3)) .or(column(Attribute.BEFORE_PREFIX + ANY_NAME_3).isEqualToText(ANY_TEXT_3)) From 1305546679bcbe6ce028e0316a3374e4c2e0723e Mon Sep 17 00:00:00 2001 From: brfrn169 Date: Fri, 20 Jun 2025 10:07:31 +0900 Subject: [PATCH 6/6] [skip ci] Add Javadoc --- .../consensuscommit/CrudHandler.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/CrudHandler.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/CrudHandler.java index 82582b6774..2aa11d6a71 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/CrudHandler.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/CrudHandler.java @@ -450,6 +450,34 @@ private Get createGet(Snapshot.Key key) { return (Get) prepareStorageSelection(buildableGet.build()); } + /** + * Waits for the completion of recovery tasks if necessary. + * + *

This method is expected to be called before committing the transaction. + * + *

We wait for the completion of recovery tasks when the recovered records are either in the + * write set or delete set, or when serializable validation is required. + * + *

This is necessary because: + * + *

    + *
  • For records in the write set or delete set, if we don’t wait for recovery tasks for them + * to complete, we might attempt to perform prepare-records on records whose status is still + * PREPARED or DELETED. + *
      + *
    • If we perform prepare-records on records that should be rolled forward, the + * prepare-records will succeed. However, it will create a PREPARED-state before + * image, which is unexpected. While this may not affect correctness, it’s something + * we should avoid. + *
    • If we perform prepare-records on records that should be rolled back, the + * prepare-records will always fail, causing the transaction to abort. + *
    + *
  • When serializable validation is required, if we don’t wait for recovery tasks to + * complete, the validation could fail due to records with PREPARED or DELETED status. + *
+ * + * @throws CrudException if any recovery task fails + */ public void waitForRecoveryCompletionIfNecessary() throws CrudException { for (RecoveryExecutor.Result recoveryResult : recoveryResults) { try {