From 960d6af9ea5620262f5b34716d889f5b793fef48 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Tue, 25 Nov 2025 09:59:39 +0900 Subject: [PATCH 1/8] Fix workflow --- .../object-storage-adapter-check.yaml | 95 ++++++++++++++++--- ...ageConditionalMutationIntegrationTest.java | 5 - ...tipleClusteringKeyScanIntegrationTest.java | 19 +++- ...geMultiplePartitionKeyIntegrationTest.java | 19 +++- ...ingleClusteringKeyScanIntegrationTest.java | 17 ++++ ...rageSinglePartitionKeyIntegrationTest.java | 17 ++++ ...ctionIntegrationTestWithObjectStorage.java | 28 ++++++ 7 files changed, 172 insertions(+), 28 deletions(-) diff --git a/.github/workflows/object-storage-adapter-check.yaml b/.github/workflows/object-storage-adapter-check.yaml index a757e7c39c..8d059141a7 100644 --- a/.github/workflows/object-storage-adapter-check.yaml +++ b/.github/workflows/object-storage-adapter-check.yaml @@ -42,25 +42,57 @@ env: INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT: '"-Dscalardb.consensus_commit.coordinator.group_commit.enabled=true" "-Dscalardb.consensus_commit.coordinator.group_commit.old_group_abort_timeout_millis=15000" --tests "**.ConsensusCommit**"' AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} - S3_REGION: ap-northeast-1 - S3_BUCKET_NAME: scalardb-test-bucket + S3_REGION: us-east-1 + S3_BUCKET_BASE_NAME: s3-scalardb-test-bucket CLOUD_STORAGE_PROJECT_ID: ${{ secrets.CLOUD_STORAGE_PROJECT_ID }} CLOUD_STORAGE_SERVICE_ACCOUNT_KEY: ${{ secrets.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }} - CLOUD_STORAGE_BUCKET_NAME: scalardb-test-bucket + CLOUD_STORAGE_BUCKET_BASE_NAME: scalardb-test-bucket jobs: integration-test-s3: - name: S3 integration test (${{ matrix.mode.label }}) + name: S3 integration test (${{ matrix.test_group.label }}) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - mode: - - label: default + test_group: + - label: consensus_commit_default + tests_filter: '--tests "**.ConsensusCommit**"' + bucket_suffix: consensus-commit group_commit_enabled: false - - label: with_group_commit + - label: consensus_commit_with_group_commit + tests_filter: '--tests "**.ConsensusCommit**"' + bucket_suffix: consensus-commit-gc group_commit_enabled: true + - label: storage_scan + tests_filter: '--tests "**.ObjectStorageSingle**" --tests "**.ObjectStorageMultiple**"' + bucket_suffix: storage-scan + group_commit_enabled: false + - label: storage_wrapper + tests_filter: '--tests "**.ObjectStorageWrapper**"' + bucket_suffix: storage-wrapper + group_commit_enabled: false + - label: storage_admin + tests_filter: '--tests "**.ObjectStorageAdmin**"' + bucket_suffix: storage-admin + group_commit_enabled: false + - label: storage_cm + tests_filter: '--tests "**.ObjectStorageConditionalMutation**"' + bucket_suffix: storage-cm + group_commit_enabled: false + - label: storage_others + tests_filter: '--tests "**.ObjectStorage**" --tests "!**.ObjectStorageSingle**" --tests "!**.ObjectStorageMultiple**" --tests "!**.ObjectStorageWrapper**" --tests "!**.ObjectStorageAdmin**" --tests "!**.ObjectStorageConditionalMutation**"' + bucket_suffix: storage + group_commit_enabled: false + - label: two_phase_consensus_commit + tests_filter: '--tests "**.TwoPhaseConsensusCommit**"' + bucket_suffix: 2pcc + group_commit_enabled: false + - label: single_crud_operation_transaction + tests_filter: '--tests "**.SingleCrudOperationTransaction**"' + bucket_suffix: single-crud + group_commit_enabled: false steps: - uses: actions/checkout@v5 @@ -95,26 +127,59 @@ jobs: uses: gradle/actions/setup-gradle@v5 - name: Execute Gradle 'integrationTestObjectStorage' task - run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=s3 -Dscalardb.object_storage.endpoint=${{ env.S3_REGION }}/${{ env.S3_BUCKET_NAME }} ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} + run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=s3 -Dscalardb.object_storage.endpoint=${{ env.S3_REGION }}/${{ env.S3_BUCKET_BASE_NAME }}-${{ matrix.test_group.bucket_suffix }} -Dscalardb.object_storage.username='${{ env.AWS_ACCESS_KEY_ID }}' -Dscalardb.object_storage.password='${{ env.AWS_SECRET_ACCESS_KEY }}' ${{ matrix.test_group.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} ${{ matrix.test_group.tests_filter }} - name: Upload Gradle test reports if: always() uses: actions/upload-artifact@v5 with: - name: s3_integration_test_reports_${{ matrix.mode.label }} + name: s3_integration_test_reports_${{ matrix.test_group.label }} path: core/build/reports/tests/integrationTestObjectStorage + integration-test-cloud-storage: - name: Cloud Storage integration test (${{ matrix.mode.label }}) + name: Cloud Storage integration test (${{ matrix.test_group.label }}) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - mode: - - label: default + test_group: + - label: consensus_commit_default + tests_filter: '--tests "**.ConsensusCommit**"' + bucket_suffix: consensus-commit group_commit_enabled: false - - label: with_group_commit + - label: consensus_commit_with_group_commit + tests_filter: '--tests "**.ConsensusCommit**"' + bucket_suffix: consensus-commit-gc group_commit_enabled: true + - label: storage_scan + tests_filter: '--tests "**.ObjectStorageSingle**" --tests "**.ObjectStorageMultiple**"' + bucket_suffix: storage-scan + group_commit_enabled: false + - label: storage_wrapper + tests_filter: '--tests "**.ObjectStorageWrapper**"' + bucket_suffix: storage-wrapper + group_commit_enabled: false + - label: storage_admin + tests_filter: '--tests "**.ObjectStorageAdmin**"' + bucket_suffix: storage-admin + group_commit_enabled: false + - label: storage_cm + tests_filter: '--tests "**.ObjectStorageConditionalMutation**"' + bucket_suffix: storage-cm + group_commit_enabled: false + - label: storage_others + tests_filter: '--tests "**.ObjectStorage**" --tests "!**.ObjectStorageSingle**" --tests "!**.ObjectStorageMultiple**" --tests "!**.ObjectStorageWrapper**" --tests "!**.ObjectStorageAdmin**" --tests "!**.ObjectStorageConditionalMutation**"' + bucket_suffix: storage + group_commit_enabled: false + - label: two_phase_consensus_commit + tests_filter: '--tests "**.TwoPhaseConsensusCommit**"' + bucket_suffix: 2pcc + group_commit_enabled: false + - label: single_crud_operation_transaction + tests_filter: '--tests "**.SingleCrudOperationTransaction**"' + bucket_suffix: single-crud + group_commit_enabled: false steps: - uses: actions/checkout@v5 @@ -149,11 +214,11 @@ jobs: uses: gradle/actions/setup-gradle@v5 - name: Execute Gradle 'integrationTestObjectStorage' task - run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=cloud-storage -Dscalardb.object_storage.endpoint=scalardb-test-bucket -Dscalardb.object_storage.username=${{ env.CLOUD_STORAGE_PROJECT_ID }} -Dscalardb.object_storage.password=${{ env.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }} ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} + run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=cloud-storage -Dscalardb.object_storage.endpoint=${{ env.CLOUD_STORAGE_BUCKET_BASE_NAME }}-${{ matrix.test_group.bucket_suffix }} -Dscalardb.object_storage.username='${{ env.CLOUD_STORAGE_PROJECT_ID }}' -Dscalardb.object_storage.password='${{ env.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }}' ${{ matrix.test_group.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} ${{ matrix.test_group.tests_filter }} - name: Upload Gradle test reports if: always() uses: actions/upload-artifact@v5 with: - name: cloud_storage_integration_test_reports_${{ matrix.mode.label }} + name: cloud_storage_integration_test_reports_${{ matrix.test_group.label }} path: core/build/reports/tests/integrationTestObjectStorage diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageConditionalMutationIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageConditionalMutationIntegrationTest.java index 3c0631a95d..199ee5c03d 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageConditionalMutationIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageConditionalMutationIntegrationTest.java @@ -6,11 +6,6 @@ public class ObjectStorageConditionalMutationIntegrationTest extends DistributedStorageConditionalMutationIntegrationTestBase { - @Override - protected int getThreadNum() { - return 3; - } - @Override protected Properties getProperties(String testName) { return ObjectStorageEnv.getProperties(testName); diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java index 5bc41617a1..bc2ac6b250 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java @@ -1,7 +1,12 @@ package com.scalar.db.storage.objectstorage; import com.scalar.db.api.DistributedStorageMultipleClusteringKeyScanIntegrationTestBase; +import com.scalar.db.io.Column; +import com.scalar.db.io.DataType; +import com.scalar.db.io.TextColumn; +import com.scalar.db.util.TestUtils; import java.util.Properties; +import java.util.stream.IntStream; public class ObjectStorageMultipleClusteringKeyScanIntegrationTest extends DistributedStorageMultipleClusteringKeyScanIntegrationTestBase { @@ -12,12 +17,18 @@ protected Properties getProperties(String testName) { } @Override - protected int getThreadNum() { - return 3; + protected boolean isParallelDdlSupported() { + return false; } @Override - protected boolean isParallelDdlSupported() { - return false; + protected Column getColumnWithMaxValue(String columnName, DataType dataType) { + if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { + // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value + StringBuilder builder = new StringBuilder(); + IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); + return TextColumn.of(columnName, builder.toString()); + } + return super.getColumnWithMaxValue(columnName, dataType); } } diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java index 7db5a06982..0ff66ce73e 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java @@ -1,7 +1,12 @@ package com.scalar.db.storage.objectstorage; import com.scalar.db.api.DistributedStorageMultiplePartitionKeyIntegrationTestBase; +import com.scalar.db.io.Column; +import com.scalar.db.io.DataType; +import com.scalar.db.io.TextColumn; +import com.scalar.db.util.TestUtils; import java.util.Properties; +import java.util.stream.IntStream; public class ObjectStorageMultiplePartitionKeyIntegrationTest extends DistributedStorageMultiplePartitionKeyIntegrationTestBase { @@ -12,12 +17,18 @@ protected Properties getProperties(String testName) { } @Override - protected int getThreadNum() { - return 3; + protected boolean isParallelDdlSupported() { + return false; } @Override - protected boolean isParallelDdlSupported() { - return false; + protected Column getColumnWithMaxValue(String columnName, DataType dataType) { + if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { + // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value + StringBuilder builder = new StringBuilder(); + IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); + return TextColumn.of(columnName, builder.toString()); + } + return super.getColumnWithMaxValue(columnName, dataType); } } diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java index 4ca86ca7e2..f848f039c9 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java @@ -1,12 +1,29 @@ package com.scalar.db.storage.objectstorage; import com.scalar.db.api.DistributedStorageSingleClusteringKeyScanIntegrationTestBase; +import com.scalar.db.io.Column; +import com.scalar.db.io.DataType; +import com.scalar.db.io.TextColumn; +import com.scalar.db.util.TestUtils; import java.util.Properties; +import java.util.stream.IntStream; public class ObjectStorageSingleClusteringKeyScanIntegrationTest extends DistributedStorageSingleClusteringKeyScanIntegrationTestBase { + @Override protected Properties getProperties(String testName) { return ObjectStorageEnv.getProperties(testName); } + + @Override + protected Column getColumnWithMaxValue(String columnName, DataType dataType) { + if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { + // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value + StringBuilder builder = new StringBuilder(); + IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); + return TextColumn.of(columnName, builder.toString()); + } + return super.getColumnWithMaxValue(columnName, dataType); + } } diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java index 5ce073100b..1d658e5d35 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java @@ -1,12 +1,29 @@ package com.scalar.db.storage.objectstorage; import com.scalar.db.api.DistributedStorageSinglePartitionKeyIntegrationTestBase; +import com.scalar.db.io.Column; +import com.scalar.db.io.DataType; +import com.scalar.db.io.TextColumn; +import com.scalar.db.util.TestUtils; import java.util.Properties; +import java.util.stream.IntStream; public class ObjectStorageSinglePartitionKeyIntegrationTest extends DistributedStorageSinglePartitionKeyIntegrationTestBase { + @Override protected Properties getProperties(String testName) { return ObjectStorageEnv.getProperties(testName); } + + @Override + protected Column getColumnWithMaxValue(String columnName, DataType dataType) { + if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { + // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value + StringBuilder builder = new StringBuilder(); + IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); + return TextColumn.of(columnName, builder.toString()); + } + return super.getColumnWithMaxValue(columnName, dataType); + } } diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionIntegrationTestWithObjectStorage.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionIntegrationTestWithObjectStorage.java index 9888104243..366cd139d8 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionIntegrationTestWithObjectStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionIntegrationTestWithObjectStorage.java @@ -1,9 +1,15 @@ package com.scalar.db.storage.objectstorage; +import com.google.common.util.concurrent.Uninterruptibles; +import com.scalar.db.api.Insert; +import com.scalar.db.api.InsertBuilder; import com.scalar.db.api.TableMetadata; +import com.scalar.db.exception.transaction.TransactionException; import com.scalar.db.io.DataType; +import com.scalar.db.io.Key; import com.scalar.db.transaction.singlecrudoperation.SingleCrudOperationTransactionIntegrationTestBase; import java.util.Properties; +import java.util.concurrent.TimeUnit; public class SingleCrudOperationTransactionIntegrationTestWithObjectStorage extends SingleCrudOperationTransactionIntegrationTestBase { @@ -30,6 +36,28 @@ protected TableMetadata getTableMetadata() { .build(); } + @Override + protected void populateRecords() throws TransactionException { + for (int i = 0; i < NUM_ACCOUNTS; i++) { + for (int j = 0; j < NUM_TYPES; j++) { + Key partitionKey = Key.ofInt(ACCOUNT_ID, i); + Key clusteringKey = Key.ofInt(ACCOUNT_TYPE, j); + InsertBuilder.Buildable insert = + Insert.newBuilder() + .namespace(namespace) + .table(TABLE) + .partitionKey(partitionKey) + .clusteringKey(clusteringKey); + prepareNonKeyColumns(i, j).forEach(insert::value); + manager.insert(insert.build()); + // Sleep to avoid Cloud Storage's request rate limit (1 write per second) + if (ObjectStorageEnv.isCloudStorage()) { + Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); + } + } + } + } + @Override protected Properties getProps(String testName) { return ObjectStorageEnv.getProperties(testName); From ff63b5af56b0fab3fbf2b659e880b23b7c9879d3 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Tue, 25 Nov 2025 16:10:19 +0900 Subject: [PATCH 2/8] Fix workflow and test --- .github/workflows/object-storage-adapter-check.yaml | 4 ++-- ...jectStorageMultipleClusteringKeyScanIntegrationTest.java | 6 ++++-- .../ObjectStorageMultiplePartitionKeyIntegrationTest.java | 6 ++++-- ...ObjectStorageSingleClusteringKeyScanIntegrationTest.java | 6 ++++-- .../ObjectStorageSinglePartitionKeyIntegrationTest.java | 6 ++++-- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/object-storage-adapter-check.yaml b/.github/workflows/object-storage-adapter-check.yaml index 8d059141a7..1a30adeca2 100644 --- a/.github/workflows/object-storage-adapter-check.yaml +++ b/.github/workflows/object-storage-adapter-check.yaml @@ -82,7 +82,7 @@ jobs: bucket_suffix: storage-cm group_commit_enabled: false - label: storage_others - tests_filter: '--tests "**.ObjectStorage**" --tests "!**.ObjectStorageSingle**" --tests "!**.ObjectStorageMultiple**" --tests "!**.ObjectStorageWrapper**" --tests "!**.ObjectStorageAdmin**" --tests "!**.ObjectStorageConditionalMutation**"' + tests_filter: '--tests "**.ObjectStorageCaseSensitivity**" --tests "**.ObjectStorageColumnValue**" --tests "**.ObjectStorageCrossPartition**" --tests "**.ObjectStorageIntegrationTest" --tests "**.ObjectStorageJapanese**" --tests "**.ObjectStorageMutationAtomicity**" --tests "**.ObjectStorageWithReservedKeyword**"' bucket_suffix: storage group_commit_enabled: false - label: two_phase_consensus_commit @@ -169,7 +169,7 @@ jobs: bucket_suffix: storage-cm group_commit_enabled: false - label: storage_others - tests_filter: '--tests "**.ObjectStorage**" --tests "!**.ObjectStorageSingle**" --tests "!**.ObjectStorageMultiple**" --tests "!**.ObjectStorageWrapper**" --tests "!**.ObjectStorageAdmin**" --tests "!**.ObjectStorageConditionalMutation**"' + tests_filter: '--tests "**.ObjectStorageCaseSensitivity**" --tests "**.ObjectStorageColumnValue**" --tests "**.ObjectStorageCrossPartition**" --tests "**.ObjectStorageIntegrationTest" --tests "**.ObjectStorageJapanese**" --tests "**.ObjectStorageMutationAtomicity**" --tests "**.ObjectStorageWithReservedKeyword**"' bucket_suffix: storage group_commit_enabled: false - label: two_phase_consensus_commit diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java index bc2ac6b250..67ab131300 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java @@ -23,8 +23,10 @@ protected boolean isParallelDdlSupported() { @Override protected Column getColumnWithMaxValue(String columnName, DataType dataType) { - if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { - // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value + if (dataType == DataType.TEXT + && (ObjectStorageEnv.isCloudStorage() || ObjectStorageEnv.isS3())) { + // Since S3 and Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the + // max value StringBuilder builder = new StringBuilder(); IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); return TextColumn.of(columnName, builder.toString()); diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java index 0ff66ce73e..04007bc121 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java @@ -23,8 +23,10 @@ protected boolean isParallelDdlSupported() { @Override protected Column getColumnWithMaxValue(String columnName, DataType dataType) { - if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { - // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value + if (dataType == DataType.TEXT + && (ObjectStorageEnv.isCloudStorage() || ObjectStorageEnv.isS3())) { + // Since S3 and Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the + // max value StringBuilder builder = new StringBuilder(); IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); return TextColumn.of(columnName, builder.toString()); diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java index f848f039c9..f4861ab65b 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java @@ -18,8 +18,10 @@ protected Properties getProperties(String testName) { @Override protected Column getColumnWithMaxValue(String columnName, DataType dataType) { - if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { - // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value + if (dataType == DataType.TEXT + && (ObjectStorageEnv.isCloudStorage() || ObjectStorageEnv.isS3())) { + // Since S3 and Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the + // max value StringBuilder builder = new StringBuilder(); IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); return TextColumn.of(columnName, builder.toString()); diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java index 1d658e5d35..d3de90f199 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java @@ -18,8 +18,10 @@ protected Properties getProperties(String testName) { @Override protected Column getColumnWithMaxValue(String columnName, DataType dataType) { - if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { - // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value + if (dataType == DataType.TEXT + && (ObjectStorageEnv.isCloudStorage() || ObjectStorageEnv.isS3())) { + // Since S3 and Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the + // max value StringBuilder builder = new StringBuilder(); IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); return TextColumn.of(columnName, builder.toString()); From 4119ccf919b106f43dc384ec504a73307626cca7 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Tue, 25 Nov 2025 20:38:53 +0900 Subject: [PATCH 3/8] Fix test --- ...torageMultipleClusteringKeyScanIntegrationTest.java | 10 ++++++++++ ...jectStorageMultiplePartitionKeyIntegrationTest.java | 10 ++++++++++ ...tStorageSingleClusteringKeyScanIntegrationTest.java | 10 ++++++++++ ...ObjectStorageSinglePartitionKeyIntegrationTest.java | 10 ++++++++++ 4 files changed, 40 insertions(+) diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java index 67ab131300..cdabf632d6 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java @@ -21,6 +21,16 @@ protected boolean isParallelDdlSupported() { return false; } + @Override + protected Column getColumnWithMinValue(String columnName, DataType dataType) { + if (dataType == DataType.TEXT && ObjectStorageEnv.isS3()) { + // Since S3 and Cloud Storage can't handle "\u0001" character correctly, we use " " as the min + // value + return TextColumn.of(columnName, " "); + } + return super.getColumnWithMinValue(columnName, dataType); + } + @Override protected Column getColumnWithMaxValue(String columnName, DataType dataType) { if (dataType == DataType.TEXT diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java index 04007bc121..591db863bc 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java @@ -21,6 +21,16 @@ protected boolean isParallelDdlSupported() { return false; } + @Override + protected Column getColumnWithMinValue(String columnName, DataType dataType) { + if (dataType == DataType.TEXT && ObjectStorageEnv.isS3()) { + // Since S3 and Cloud Storage can't handle "\u0001" character correctly, we use " " as the min + // value + return TextColumn.of(columnName, " "); + } + return super.getColumnWithMinValue(columnName, dataType); + } + @Override protected Column getColumnWithMaxValue(String columnName, DataType dataType) { if (dataType == DataType.TEXT diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java index f4861ab65b..dda12159a8 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java @@ -16,6 +16,16 @@ protected Properties getProperties(String testName) { return ObjectStorageEnv.getProperties(testName); } + @Override + protected Column getColumnWithMinValue(String columnName, DataType dataType) { + if (dataType == DataType.TEXT && ObjectStorageEnv.isS3()) { + // Since S3 and Cloud Storage can't handle "\u0001" character correctly, we use " " as the min + // value + return TextColumn.of(columnName, " "); + } + return super.getColumnWithMinValue(columnName, dataType); + } + @Override protected Column getColumnWithMaxValue(String columnName, DataType dataType) { if (dataType == DataType.TEXT diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java index d3de90f199..669d344145 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java @@ -16,6 +16,16 @@ protected Properties getProperties(String testName) { return ObjectStorageEnv.getProperties(testName); } + @Override + protected Column getColumnWithMinValue(String columnName, DataType dataType) { + if (dataType == DataType.TEXT && ObjectStorageEnv.isS3()) { + // Since S3 and Cloud Storage can't handle "\u0001" character correctly, we use " " as the min + // value + return TextColumn.of(columnName, " "); + } + return super.getColumnWithMinValue(columnName, dataType); + } + @Override protected Column getColumnWithMaxValue(String columnName, DataType dataType) { if (dataType == DataType.TEXT From 94af09453325eb0d5db60eb5bd4e59b819ad1fa0 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Wed, 26 Nov 2025 08:40:18 +0900 Subject: [PATCH 4/8] Fix workflow and test --- .../object-storage-adapter-check.yaml | 20 +++++++++++++------ ...tipleClusteringKeyScanIntegrationTest.java | 19 ++++++------------ ...geMultiplePartitionKeyIntegrationTest.java | 19 ++++++------------ ...ingleClusteringKeyScanIntegrationTest.java | 16 ++------------- ...rageSinglePartitionKeyIntegrationTest.java | 16 ++------------- 5 files changed, 30 insertions(+), 60 deletions(-) diff --git a/.github/workflows/object-storage-adapter-check.yaml b/.github/workflows/object-storage-adapter-check.yaml index 1a30adeca2..ae4435dd5c 100644 --- a/.github/workflows/object-storage-adapter-check.yaml +++ b/.github/workflows/object-storage-adapter-check.yaml @@ -65,9 +65,13 @@ jobs: tests_filter: '--tests "**.ConsensusCommit**"' bucket_suffix: consensus-commit-gc group_commit_enabled: true - - label: storage_scan - tests_filter: '--tests "**.ObjectStorageSingle**" --tests "**.ObjectStorageMultiple**"' - bucket_suffix: storage-scan + - label: storage_scan_single + tests_filter: '--tests "**.ObjectStorageSingle**"' + bucket_suffix: storage-scan-single + group_commit_enabled: false + - label: storage_scan_multiple + tests_filter: '--tests "**.ObjectStorageMultiple**"' + bucket_suffix: storage-scan-multiple group_commit_enabled: false - label: storage_wrapper tests_filter: '--tests "**.ObjectStorageWrapper**"' @@ -152,9 +156,13 @@ jobs: tests_filter: '--tests "**.ConsensusCommit**"' bucket_suffix: consensus-commit-gc group_commit_enabled: true - - label: storage_scan - tests_filter: '--tests "**.ObjectStorageSingle**" --tests "**.ObjectStorageMultiple**"' - bucket_suffix: storage-scan + - label: storage_scan_single + tests_filter: '--tests "**.ObjectStorageSingle**"' + bucket_suffix: storage-scan-single + group_commit_enabled: false + - label: storage_scan_multiple + tests_filter: '--tests "**.ObjectStorageMultiple**"' + bucket_suffix: storage-scan-multiple group_commit_enabled: false - label: storage_wrapper tests_filter: '--tests "**.ObjectStorageWrapper**"' diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java index cdabf632d6..3316fe144a 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java @@ -17,26 +17,19 @@ protected Properties getProperties(String testName) { } @Override - protected boolean isParallelDdlSupported() { - return false; + protected int getThreadNum() { + return 3; } @Override - protected Column getColumnWithMinValue(String columnName, DataType dataType) { - if (dataType == DataType.TEXT && ObjectStorageEnv.isS3()) { - // Since S3 and Cloud Storage can't handle "\u0001" character correctly, we use " " as the min - // value - return TextColumn.of(columnName, " "); - } - return super.getColumnWithMinValue(columnName, dataType); + protected boolean isParallelDdlSupported() { + return false; } @Override protected Column getColumnWithMaxValue(String columnName, DataType dataType) { - if (dataType == DataType.TEXT - && (ObjectStorageEnv.isCloudStorage() || ObjectStorageEnv.isS3())) { - // Since S3 and Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the - // max value + if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { + // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value StringBuilder builder = new StringBuilder(); IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); return TextColumn.of(columnName, builder.toString()); diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java index 591db863bc..1481f54032 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java @@ -17,26 +17,19 @@ protected Properties getProperties(String testName) { } @Override - protected boolean isParallelDdlSupported() { - return false; + protected int getThreadNum() { + return 3; } @Override - protected Column getColumnWithMinValue(String columnName, DataType dataType) { - if (dataType == DataType.TEXT && ObjectStorageEnv.isS3()) { - // Since S3 and Cloud Storage can't handle "\u0001" character correctly, we use " " as the min - // value - return TextColumn.of(columnName, " "); - } - return super.getColumnWithMinValue(columnName, dataType); + protected boolean isParallelDdlSupported() { + return false; } @Override protected Column getColumnWithMaxValue(String columnName, DataType dataType) { - if (dataType == DataType.TEXT - && (ObjectStorageEnv.isCloudStorage() || ObjectStorageEnv.isS3())) { - // Since S3 and Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the - // max value + if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { + // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value StringBuilder builder = new StringBuilder(); IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); return TextColumn.of(columnName, builder.toString()); diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java index dda12159a8..f848f039c9 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java @@ -16,22 +16,10 @@ protected Properties getProperties(String testName) { return ObjectStorageEnv.getProperties(testName); } - @Override - protected Column getColumnWithMinValue(String columnName, DataType dataType) { - if (dataType == DataType.TEXT && ObjectStorageEnv.isS3()) { - // Since S3 and Cloud Storage can't handle "\u0001" character correctly, we use " " as the min - // value - return TextColumn.of(columnName, " "); - } - return super.getColumnWithMinValue(columnName, dataType); - } - @Override protected Column getColumnWithMaxValue(String columnName, DataType dataType) { - if (dataType == DataType.TEXT - && (ObjectStorageEnv.isCloudStorage() || ObjectStorageEnv.isS3())) { - // Since S3 and Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the - // max value + if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { + // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value StringBuilder builder = new StringBuilder(); IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); return TextColumn.of(columnName, builder.toString()); diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java index 669d344145..1d658e5d35 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java @@ -16,22 +16,10 @@ protected Properties getProperties(String testName) { return ObjectStorageEnv.getProperties(testName); } - @Override - protected Column getColumnWithMinValue(String columnName, DataType dataType) { - if (dataType == DataType.TEXT && ObjectStorageEnv.isS3()) { - // Since S3 and Cloud Storage can't handle "\u0001" character correctly, we use " " as the min - // value - return TextColumn.of(columnName, " "); - } - return super.getColumnWithMinValue(columnName, dataType); - } - @Override protected Column getColumnWithMaxValue(String columnName, DataType dataType) { - if (dataType == DataType.TEXT - && (ObjectStorageEnv.isCloudStorage() || ObjectStorageEnv.isS3())) { - // Since S3 and Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the - // max value + if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) { + // Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value StringBuilder builder = new StringBuilder(); IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z')); return TextColumn.of(columnName, builder.toString()); From 517675ad3151f0cc6976acd4673003667a447bc9 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Wed, 26 Nov 2025 19:22:03 +0900 Subject: [PATCH 5/8] Add waits --- ...ommitIntegrationTestWithObjectStorage.java | 21 ++++++++++++++++ ...cificIntegrationTestWithObjectStorage.java | 25 +++++++++++++++++++ ...nsusCommitSpecificIntegrationTestBase.java | 8 +++--- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java index 309216210a..7888648fc1 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java @@ -1,14 +1,35 @@ package com.scalar.db.storage.objectstorage; +import com.google.common.util.concurrent.Uninterruptibles; import com.scalar.db.api.TableMetadata; import com.scalar.db.io.DataType; import com.scalar.db.transaction.consensuscommit.ConsensusCommitIntegrationTestBase; import java.util.Properties; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; public class ConsensusCommitIntegrationTestWithObjectStorage extends ConsensusCommitIntegrationTestBase { + @Override + @BeforeEach + public void setUp() throws Exception { + admin.truncateTable(namespace, TABLE); + admin.truncateCoordinatorTables(); + if (ObjectStorageEnv.isCloudStorage()) { + Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); + } + } + + @AfterEach + void tearDown() { + if (ObjectStorageEnv.isCloudStorage()) { + Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); + } + } + @Override protected TableMetadata getTableMetadata() { return TableMetadata.newBuilder() diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java index 640d0252bb..c3b7491c75 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java @@ -1,15 +1,40 @@ package com.scalar.db.storage.objectstorage; +import com.google.common.util.concurrent.Uninterruptibles; import com.scalar.db.api.TableMetadata; import com.scalar.db.io.DataType; import com.scalar.db.transaction.consensuscommit.ConsensusCommitSpecificIntegrationTestBase; import com.scalar.db.transaction.consensuscommit.Isolation; import java.util.Properties; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; public class ConsensusCommitSpecificIntegrationTestWithObjectStorage extends ConsensusCommitSpecificIntegrationTestBase { + @Override + @BeforeEach + protected void setUp() throws Exception { + truncateTables(); + if (ObjectStorageEnv.isCloudStorage()) { + Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); + } + } + + @Override + @AfterEach + public void tearDown() { + recoveryExecutor.close(); + if (groupCommitter != null) { + groupCommitter.close(); + } + if (ObjectStorageEnv.isCloudStorage()) { + Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); + } + } + @Override protected TableMetadata getTableMetadata() { return TableMetadata.newBuilder() diff --git a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java index 716533854d..6c5993ff90 100644 --- a/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitSpecificIntegrationTestBase.java @@ -112,9 +112,9 @@ public abstract class ConsensusCommitSpecificIntegrationTestBase { private DistributedStorage storage; private Coordinator coordinator; private RecoveryHandler recovery; - private RecoveryExecutor recoveryExecutor; + protected RecoveryExecutor recoveryExecutor; private CommitHandler commit; - @Nullable private CoordinatorGroupCommitter groupCommitter; + @Nullable protected CoordinatorGroupCommitter groupCommitter; @BeforeAll void beforeAll() throws Exception { @@ -174,7 +174,7 @@ protected Map getCreationOptions() { } @BeforeEach - void setUp() throws Exception { + protected void setUp() throws Exception { truncateTables(); } @@ -186,7 +186,7 @@ public void tearDown() { } } - private void truncateTables() throws ExecutionException { + protected void truncateTables() throws ExecutionException { consensusCommitAdmin.truncateTable(namespace1, TABLE_1); consensusCommitAdmin.truncateTable(namespace2, TABLE_2); consensusCommitAdmin.truncateCoordinatorTables(); From 0c3da519142e6aeb54292d90fca1b921d55a0a98 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Thu, 27 Nov 2025 12:03:36 +0900 Subject: [PATCH 6/8] Prepare more buckets --- .../object-storage-adapter-check.yaml | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/.github/workflows/object-storage-adapter-check.yaml b/.github/workflows/object-storage-adapter-check.yaml index ae4435dd5c..d8b5ca2839 100644 --- a/.github/workflows/object-storage-adapter-check.yaml +++ b/.github/workflows/object-storage-adapter-check.yaml @@ -58,13 +58,29 @@ jobs: matrix: test_group: - label: consensus_commit_default - tests_filter: '--tests "**.ConsensusCommit**"' + tests_filter: '--tests "**.ConsensusCommitIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitNullMetadataIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitWithIncludeMetadataEnabledIntegrationTestWithObjectStorage"' bucket_suffix: consensus-commit group_commit_enabled: false - label: consensus_commit_with_group_commit - tests_filter: '--tests "**.ConsensusCommit**"' + tests_filter: '--tests "**.ConsensusCommitIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitNullMetadataIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitWithIncludeMetadataEnabledIntegrationTestWithObjectStorage"' bucket_suffix: consensus-commit-gc group_commit_enabled: true + - label: consensus_commit_admin + tests_filter: '--tests "**.ConsensusCommitAdminIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitAdminRepairIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-admin + group_commit_enabled: false + - label: consensus_commit_admin_with_group_commit + tests_filter: '--tests "**.ConsensusCommitAdminIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitAdminRepairIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-admin-gc + group_commit_enabled: true + - label: consensus_commit_specific + tests_filter: '--tests "**.ConsensusCommitSpecificIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-specific + group_commit_enabled: false + - label: consensus_commit_specific_with_group_commit + tests_filter: '--tests "**.ConsensusCommitSpecificIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-specific-gc + group_commit_enabled: true - label: storage_scan_single tests_filter: '--tests "**.ObjectStorageSingle**"' bucket_suffix: storage-scan-single @@ -149,13 +165,29 @@ jobs: matrix: test_group: - label: consensus_commit_default - tests_filter: '--tests "**.ConsensusCommit**"' + tests_filter: '--tests "**.ConsensusCommitIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitNullMetadataIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitWithIncludeMetadataEnabledIntegrationTestWithObjectStorage"' bucket_suffix: consensus-commit group_commit_enabled: false - label: consensus_commit_with_group_commit - tests_filter: '--tests "**.ConsensusCommit**"' + tests_filter: '--tests "**.ConsensusCommitIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitNullMetadataIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitWithIncludeMetadataEnabledIntegrationTestWithObjectStorage"' bucket_suffix: consensus-commit-gc group_commit_enabled: true + - label: consensus_commit_admin + tests_filter: '--tests "**.ConsensusCommitAdminIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitAdminRepairIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-admin + group_commit_enabled: false + - label: consensus_commit_admin_with_group_commit + tests_filter: '--tests "**.ConsensusCommitAdminIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitAdminRepairIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-admin-gc + group_commit_enabled: true + - label: consensus_commit_specific + tests_filter: '--tests "**.ConsensusCommitSpecificIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-specific + group_commit_enabled: false + - label: consensus_commit_specific_with_group_commit + tests_filter: '--tests "**.ConsensusCommitSpecificIntegrationTestWithObjectStorage"' + bucket_suffix: consensus-commit-specific-gc + group_commit_enabled: true - label: storage_scan_single tests_filter: '--tests "**.ObjectStorageSingle**"' bucket_suffix: storage-scan-single From ee5da61c427f0051d05f5417bed194d4e68e01de Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Thu, 27 Nov 2025 22:50:44 +0900 Subject: [PATCH 7/8] Fix test --- .../ConsensusCommitIntegrationTestWithObjectStorage.java | 4 ++-- ...nsensusCommitSpecificIntegrationTestWithObjectStorage.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java index 7888648fc1..49f1cbbd22 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java @@ -19,14 +19,14 @@ public void setUp() throws Exception { admin.truncateTable(namespace, TABLE); admin.truncateCoordinatorTables(); if (ObjectStorageEnv.isCloudStorage()) { - Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); } } @AfterEach void tearDown() { if (ObjectStorageEnv.isCloudStorage()) { - Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); } } diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java index c3b7491c75..858e1206c2 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java @@ -19,7 +19,7 @@ public class ConsensusCommitSpecificIntegrationTestWithObjectStorage protected void setUp() throws Exception { truncateTables(); if (ObjectStorageEnv.isCloudStorage()) { - Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); } } @@ -31,7 +31,7 @@ public void tearDown() { groupCommitter.close(); } if (ObjectStorageEnv.isCloudStorage()) { - Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); } } From f5a9b9df330a63856d65026814c9fc0e1a6bbc84 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Fri, 28 Nov 2025 11:55:56 +0900 Subject: [PATCH 8/8] Apply suggestions from code review --- .../ConsensusCommitIntegrationTestWithObjectStorage.java | 5 +++-- ...usCommitSpecificIntegrationTestWithObjectStorage.java | 9 ++++----- ...ationTransactionIntegrationTestWithObjectStorage.java | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java index 49f1cbbd22..4029e07dcd 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java @@ -16,9 +16,9 @@ public class ConsensusCommitIntegrationTestWithObjectStorage @Override @BeforeEach public void setUp() throws Exception { - admin.truncateTable(namespace, TABLE); - admin.truncateCoordinatorTables(); + super.setUp(); if (ObjectStorageEnv.isCloudStorage()) { + // Sleep to mitigate rate limit errors Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); } } @@ -26,6 +26,7 @@ public void setUp() throws Exception { @AfterEach void tearDown() { if (ObjectStorageEnv.isCloudStorage()) { + // Sleep to mitigate rate limit errors Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); } } diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java index 858e1206c2..6f1720e3ec 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java @@ -17,8 +17,9 @@ public class ConsensusCommitSpecificIntegrationTestWithObjectStorage @Override @BeforeEach protected void setUp() throws Exception { - truncateTables(); + super.setUp(); if (ObjectStorageEnv.isCloudStorage()) { + // Sleep to mitigate rate limit errors Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); } } @@ -26,11 +27,9 @@ protected void setUp() throws Exception { @Override @AfterEach public void tearDown() { - recoveryExecutor.close(); - if (groupCommitter != null) { - groupCommitter.close(); - } + super.tearDown(); if (ObjectStorageEnv.isCloudStorage()) { + // Sleep to mitigate rate limit errors Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); } } diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionIntegrationTestWithObjectStorage.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionIntegrationTestWithObjectStorage.java index 366cd139d8..b0f7d0f5cb 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionIntegrationTestWithObjectStorage.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionIntegrationTestWithObjectStorage.java @@ -50,8 +50,8 @@ protected void populateRecords() throws TransactionException { .clusteringKey(clusteringKey); prepareNonKeyColumns(i, j).forEach(insert::value); manager.insert(insert.build()); - // Sleep to avoid Cloud Storage's request rate limit (1 write per second) if (ObjectStorageEnv.isCloudStorage()) { + // Sleep to mitigate rate limit errors Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); } }