diff --git a/.github/workflows/object-storage-adapter-check.yaml b/.github/workflows/object-storage-adapter-check.yaml index 3c9932abed..69f79cbb36 100644 --- a/.github/workflows/object-storage-adapter-check.yaml +++ b/.github/workflows/object-storage-adapter-check.yaml @@ -42,25 +42,77 @@ env: INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT: '"-Dscalardb.consensus_commit.coordinator.group_commit.enabled=true" "-Dscalardb.consensus_commit.coordinator.group_commit.old_group_abort_timeout_millis=15000" --tests "**.ConsensusCommit**"' AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} - S3_REGION: ap-northeast-1 - S3_BUCKET_NAME: scalardb-test-bucket + S3_REGION: us-east-1 + S3_BUCKET_BASE_NAME: s3-scalardb-test-bucket CLOUD_STORAGE_PROJECT_ID: ${{ secrets.CLOUD_STORAGE_PROJECT_ID }} CLOUD_STORAGE_SERVICE_ACCOUNT_KEY: ${{ secrets.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }} - CLOUD_STORAGE_BUCKET_NAME: scalardb-test-bucket + CLOUD_STORAGE_BUCKET_BASE_NAME: scalardb-test-bucket jobs: integration-test-s3: - name: S3 integration test (${{ matrix.mode.label }}) + name: S3 integration test (${{ matrix.test_group.label }}) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - mode: - - label: default + test_group: + - label: consensus_commit_default + tests_filter: '--tests "**.ConsensusCommitIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitNullMetadataIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitWithIncludeMetadataEnabledIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit group_commit_enabled: false - - label: with_group_commit + - label: consensus_commit_with_group_commit + tests_filter: '--tests "**.ConsensusCommitIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitNullMetadataIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitWithIncludeMetadataEnabledIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-gc group_commit_enabled: true + - label: consensus_commit_admin + tests_filter: '--tests "**.ConsensusCommitAdminIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitAdminRepairIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-admin + group_commit_enabled: false + - label: consensus_commit_admin_with_group_commit + tests_filter: '--tests "**.ConsensusCommitAdminIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitAdminRepairIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-admin-gc + group_commit_enabled: true + - label: consensus_commit_specific + tests_filter: '--tests "**.ConsensusCommitSpecificIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-specific + group_commit_enabled: false + - label: consensus_commit_specific_with_group_commit + tests_filter: '--tests "**.ConsensusCommitSpecificIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-specific-gc + group_commit_enabled: true + - label: storage_scan_single + tests_filter: '--tests "**.ObjectStorageSingle**"' + bucket_suffix: storage-scan-single + group_commit_enabled: false + - label: storage_scan_multiple + tests_filter: '--tests "**.ObjectStorageMultiple**"' + bucket_suffix: storage-scan-multiple + group_commit_enabled: false + - label: storage_wrapper + tests_filter: '--tests "**.ObjectStorageWrapper**"' + bucket_suffix: storage-wrapper + group_commit_enabled: false + - label: storage_admin + tests_filter: '--tests "**.ObjectStorageAdmin**"' + bucket_suffix: storage-admin + group_commit_enabled: false + - label: storage_cm + tests_filter: '--tests "**.ObjectStorageConditionalMutation**"' + bucket_suffix: storage-cm + group_commit_enabled: false + - label: storage_others + tests_filter: '--tests "**.ObjectStorageCaseSensitivity**" --tests "**.ObjectStorageColumnValue**" --tests "**.ObjectStorageCrossPartition**" --tests "**.ObjectStorageIntegrationTest" --tests "**.ObjectStorageJapanese**" --tests "**.ObjectStorageMutationAtomicity**" --tests "**.ObjectStorageWithReservedKeyword**"' + bucket_suffix: storage + group_commit_enabled: false + - label: two_phase_consensus_commit + tests_filter: '--tests "**.TwoPhaseConsensusCommit**"' + bucket_suffix: 2pcc + group_commit_enabled: false + - label: single_crud_operation_transaction + tests_filter: '--tests "**.SingleCrudOperationTransaction**"' + bucket_suffix: single-crud + group_commit_enabled: false steps: - uses: actions/checkout@v6 @@ -95,26 +147,79 @@ jobs: uses: gradle/actions/setup-gradle@v5 - name: Execute Gradle 'integrationTestObjectStorage' task - run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=s3 -Dscalardb.object_storage.endpoint=${{ env.S3_REGION }}/${{ env.S3_BUCKET_NAME }} ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} + run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=s3 -Dscalardb.object_storage.endpoint=${{ env.S3_REGION }}/${{ env.S3_BUCKET_BASE_NAME }}-${{ matrix.test_group.bucket_suffix }} -Dscalardb.object_storage.username='${{ env.AWS_ACCESS_KEY_ID }}' -Dscalardb.object_storage.password='${{ env.AWS_SECRET_ACCESS_KEY }}' ${{ matrix.test_group.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} ${{ matrix.test_group.tests_filter }} - name: Upload Gradle test reports if: always() uses: actions/upload-artifact@v5 with: - name: s3_integration_test_reports_${{ matrix.mode.label }} + name: s3_integration_test_reports_${{ matrix.test_group.label }} path: core/build/reports/tests/integrationTestObjectStorage + integration-test-cloud-storage: - name: Cloud Storage integration test (${{ matrix.mode.label }}) + name: Cloud Storage integration test (${{ matrix.test_group.label }}) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - mode: - - label: default + test_group: + - label: consensus_commit_default + tests_filter: '--tests "**.ConsensusCommitIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitNullMetadataIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitWithIncludeMetadataEnabledIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit group_commit_enabled: false - - label: with_group_commit + - label: consensus_commit_with_group_commit + tests_filter: '--tests "**.ConsensusCommitIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitNullMetadataIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitWithIncludeMetadataEnabledIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-gc group_commit_enabled: true + - label: consensus_commit_admin + tests_filter: '--tests "**.ConsensusCommitAdminIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitAdminRepairIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-admin + group_commit_enabled: false + - label: consensus_commit_admin_with_group_commit + tests_filter: '--tests "**.ConsensusCommitAdminIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitAdminRepairIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-admin-gc + group_commit_enabled: true + - label: consensus_commit_specific + tests_filter: '--tests "**.ConsensusCommitSpecificIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-specific + group_commit_enabled: false + - label: consensus_commit_specific_with_group_commit + tests_filter: '--tests "**.ConsensusCommitSpecificIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-specific-gc + group_commit_enabled: true + - label: storage_scan_single + tests_filter: '--tests "**.ObjectStorageSingle**"' + bucket_suffix: storage-scan-single + group_commit_enabled: false + - label: storage_scan_multiple + tests_filter: '--tests "**.ObjectStorageMultiple**"' + bucket_suffix: storage-scan-multiple + group_commit_enabled: false + - label: storage_wrapper + tests_filter: '--tests "**.ObjectStorageWrapper**"' + bucket_suffix: storage-wrapper + group_commit_enabled: false + - label: storage_admin + tests_filter: '--tests "**.ObjectStorageAdmin**"' + bucket_suffix: storage-admin + group_commit_enabled: false + - label: storage_cm + tests_filter: '--tests "**.ObjectStorageConditionalMutation**"' + bucket_suffix: storage-cm + group_commit_enabled: false + - label: storage_others + tests_filter: '--tests "**.ObjectStorageCaseSensitivity**" --tests "**.ObjectStorageColumnValue**" --tests "**.ObjectStorageCrossPartition**" --tests "**.ObjectStorageIntegrationTest" --tests "**.ObjectStorageJapanese**" --tests "**.ObjectStorageMutationAtomicity**" --tests "**.ObjectStorageWithReservedKeyword**"' + bucket_suffix: storage + group_commit_enabled: false + - label: two_phase_consensus_commit + tests_filter: '--tests "**.TwoPhaseConsensusCommit**"' + bucket_suffix: 2pcc + group_commit_enabled: false + - label: single_crud_operation_transaction + tests_filter: '--tests "**.SingleCrudOperationTransaction**"' + bucket_suffix: single-crud + group_commit_enabled: false steps: - uses: actions/checkout@v6 @@ -149,11 +254,11 @@ jobs: uses: gradle/actions/setup-gradle@v5 - name: Execute Gradle 'integrationTestObjectStorage' task - run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=cloud-storage -Dscalardb.object_storage.endpoint=scalardb-test-bucket -Dscalardb.object_storage.username=${{ env.CLOUD_STORAGE_PROJECT_ID }} -Dscalardb.object_storage.password=${{ env.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }} ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} + run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=cloud-storage -Dscalardb.object_storage.endpoint=${{ env.CLOUD_STORAGE_BUCKET_BASE_NAME }}-${{ matrix.test_group.bucket_suffix }} -Dscalardb.object_storage.username='${{ env.CLOUD_STORAGE_PROJECT_ID }}' -Dscalardb.object_storage.password='${{ env.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }}' ${{ matrix.test_group.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} ${{ matrix.test_group.tests_filter }} - name: Upload Gradle test reports if: always() uses: actions/upload-artifact@v5 with: - name: cloud_storage_integration_test_reports_${{ matrix.mode.label }} + name: cloud_storage_integration_test_reports_${{ matrix.test_group.label }} path: core/build/reports/tests/integrationTestObjectStorage diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java index 309216210a..4029e07dcd 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java @@ -1,14 +1,36 @@ package com.scalar.db.storage.objectstorage; +import com.google.common.util.concurrent.Uninterruptibles; import com.scalar.db.api.TableMetadata; import com.scalar.db.io.DataType; import com.scalar.db.transaction.consensuscommit.ConsensusCommitIntegrationTestBase; import java.util.Properties; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; public class ConsensusCommitIntegrationTestWithObjectStorage extends ConsensusCommitIntegrationTestBase { + @Override + @BeforeEach + public void setUp() throws Exception { + super.setUp(); + if (ObjectStorageEnv.isCloudStorage()) { + // Sleep to mitigate rate limit errors + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + } + } + + @AfterEach + void tearDown() { + if (ObjectStorageEnv.isCloudStorage()) { + // Sleep to mitigate rate limit errors + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + } + } + @Override protected TableMetadata getTableMetadata() { return TableMetadata.newBuilder() diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java index 640d0252bb..6f1720e3ec 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java @@ -1,15 +1,39 @@ package com.scalar.db.storage.objectstorage; +import com.google.common.util.concurrent.Uninterruptibles; import com.scalar.db.api.TableMetadata; import com.scalar.db.io.DataType; import com.scalar.db.transaction.consensuscommit.ConsensusCommitSpecificIntegrationTestBase; import com.scalar.db.transaction.consensuscommit.Isolation; import java.util.Properties; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; public class ConsensusCommitSpecificIntegrationTestWithObjectStorage extends ConsensusCommitSpecificIntegrationTestBase { + @Override + @BeforeEach + protected void setUp() throws Exception { + super.setUp(); + if (ObjectStorageEnv.isCloudStorage()) { + // Sleep to mitigate rate limit errors + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + } + } + + @Override + @AfterEach + public void tearDown() { + super.tearDown(); + if (ObjectStorageEnv.isCloudStorage()) { + // Sleep to mitigate rate limit errors + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + } + } + @Override protected TableMetadata getTableMetadata() { return TableMetadata.newBuilder() diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageConditionalMutationIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageConditionalMutationIntegrationTest.java index 3c0631a95d..199ee5c03d 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageConditionalMutationIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageConditionalMutationIntegrationTest.java @@ -6,11 +6,6 @@ public class ObjectStorageConditionalMutationIntegrationTest extends DistributedStorageConditionalMutationIntegrationTestBase { - @Override - protected int getThreadNum() { - return 3; - } - @Override protected Properties getProperties(String testName) { return ObjectStorageEnv.getProperties(testName); diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java index 5bc41617a1..3316fe144a 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java @@ -1,7 +1,12 @@ package com.scalar.db.storage.objectstorage; import com.scalar.db.api.DistributedStorageMultipleClusteringKeyScanIntegrationTestBase; +import com.scalar.db.io.Column; +import com.scalar.db.io.DataType; +import com.scalar.db.io.TextColumn; +import com.scalar.db.util.TestUtils; import java.util.Properties; +import java.util.stream.IntStream; public class ObjectStorageMultipleClusteringKeyScanIntegrationTest extends DistributedStorageMultipleClusteringKeyScanIntegrationTestBase { @@ -20,4 +25,15 @@ protected int getThreadNum() { protected boolean isParallelDdlSupported() { return false; } + + @Override + protected Column getColumnWithMaxValue(String columnName, DataType dataType) { + if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { + // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value + StringBuilder builder = new StringBuilder(); + IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); + return TextColumn.of(columnName, builder.toString()); + } + return super.getColumnWithMaxValue(columnName, dataType); + } } diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java index 7db5a06982..1481f54032 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java @@ -1,7 +1,12 @@ package com.scalar.db.storage.objectstorage; import com.scalar.db.api.DistributedStorageMultiplePartitionKeyIntegrationTestBase; +import com.scalar.db.io.Column; +import com.scalar.db.io.DataType; +import com.scalar.db.io.TextColumn; +import com.scalar.db.util.TestUtils; import java.util.Properties; +import java.util.stream.IntStream; public class ObjectStorageMultiplePartitionKeyIntegrationTest extends DistributedStorageMultiplePartitionKeyIntegrationTestBase { @@ -20,4 +25,15 @@ protected int getThreadNum() { protected boolean isParallelDdlSupported() { return false; } + + @Override + protected Column getColumnWithMaxValue(String columnName, DataType dataType) { + if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { + // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value + StringBuilder builder = new StringBuilder(); + IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); + return TextColumn.of(columnName, builder.toString()); + } + return super.getColumnWithMaxValue(columnName, dataType); + } } diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java index 4ca86ca7e2..f848f039c9 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java @@ -1,12 +1,29 @@ package com.scalar.db.storage.objectstorage; import com.scalar.db.api.DistributedStorageSingleClusteringKeyScanIntegrationTestBase; +import com.scalar.db.io.Column; +import com.scalar.db.io.DataType; +import com.scalar.db.io.TextColumn; +import com.scalar.db.util.TestUtils; import java.util.Properties; +import java.util.stream.IntStream; public class ObjectStorageSingleClusteringKeyScanIntegrationTest extends DistributedStorageSingleClusteringKeyScanIntegrationTestBase { + @Override protected Properties getProperties(String testName) { return ObjectStorageEnv.getProperties(testName); } + + @Override + protected Column getColumnWithMaxValue(String columnName, DataType dataType) { + if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { + // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value + StringBuilder builder = new StringBuilder(); + IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); + return TextColumn.of(columnName, builder.toString()); + } + return super.getColumnWithMaxValue(columnName, dataType); + } } diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java index 5ce073100b..1d658e5d35 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java @@ -1,12 +1,29 @@ package com.scalar.db.storage.objectstorage; import com.scalar.db.api.DistributedStorageSinglePartitionKeyIntegrationTestBase; +import com.scalar.db.io.Column; +import com.scalar.db.io.DataType; +import com.scalar.db.io.TextColumn; +import com.scalar.db.util.TestUtils; import java.util.Properties; +import java.util.stream.IntStream; public class ObjectStorageSinglePartitionKeyIntegrationTest extends DistributedStorageSinglePartitionKeyIntegrationTestBase { + @Override protected Properties getProperties(String testName) { return ObjectStorageEnv.getProperties(testName); } + + @Override + protected Column getColumnWithMaxValue(String columnName, DataType dataType) { + if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { + // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value + StringBuilder builder = new StringBuilder(); + IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); + return TextColumn.of(columnName, builder.toString()); + } + return super.getColumnWithMaxValue(columnName, dataType); + } } diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionIntegrationTestWithObjectStorage.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionIntegrationTestWithObjectStorage.java index 9888104243..b0f7d0f5cb 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionIntegrationTestWithObjectStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionIntegrationTestWithObjectStorage.java @@ -1,9 +1,15 @@ package com.scalar.db.storage.objectstorage; +import com.google.common.util.concurrent.Uninterruptibles; +import com.scalar.db.api.Insert; +import com.scalar.db.api.InsertBuilder; import com.scalar.db.api.TableMetadata; +import com.scalar.db.exception.transaction.TransactionException; import com.scalar.db.io.DataType; +import com.scalar.db.io.Key; import com.scalar.db.transaction.singlecrudoperation.SingleCrudOperationTransactionIntegrationTestBase; import java.util.Properties; +import java.util.concurrent.TimeUnit; public class SingleCrudOperationTransactionIntegrationTestWithObjectStorage extends SingleCrudOperationTransactionIntegrationTestBase { @@ -30,6 +36,28 @@ protected TableMetadata getTableMetadata() { .build(); } + @Override + protected void populateRecords() throws TransactionException { + for (int i = 0; i < NUM_ACCOUNTS; i++) { + for (int j = 0; j < NUM_TYPES; j++) { + Key partitionKey = Key.ofInt(ACCOUNT_ID, i); + Key clusteringKey = Key.ofInt(ACCOUNT_TYPE, j); + InsertBuilder.Buildable insert = + Insert.newBuilder() + .namespace(namespace) + .table(TABLE) + .partitionKey(partitionKey) + .clusteringKey(clusteringKey); + prepareNonKeyColumns(i, j).forEach(insert::value); + manager.insert(insert.build()); + if (ObjectStorageEnv.isCloudStorage()) { + // Sleep to mitigate rate limit errors + Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); + } + } + } + } + @Override protected Properties getProps(String testName) { return ObjectStorageEnv.getProperties(testName); diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java index 01382d7cac..d756ced12f 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java @@ -111,9 +111,9 @@ public abstract class ConsensusCommitSpecificIntegrationTestBase { private DistributedStorage storage; private Coordinator coordinator; private RecoveryHandler recovery; - private RecoveryExecutor recoveryExecutor; + protected RecoveryExecutor recoveryExecutor; private CommitHandler commit; - @Nullable private CoordinatorGroupCommitter groupCommitter; + @Nullable protected CoordinatorGroupCommitter groupCommitter; @BeforeAll void beforeAll() throws Exception { @@ -176,7 +176,7 @@ protected Map getCreationOptions() { } @BeforeEach - void setUp() throws Exception { + protected void setUp() throws Exception { truncateTables(); } @@ -188,7 +188,7 @@ public void tearDown() { } } - private void truncateTables() throws ExecutionException { + protected void truncateTables() throws ExecutionException { consensusCommitAdmin.truncateTable(namespace1, TABLE_1); consensusCommitAdmin.truncateTable(namespace2, TABLE_2); consensusCommitAdmin.truncateCoordinatorTables();