Skip to content

Commit 960d6af

Browse files
KodaiDKodaiD
committed
Fix workflow
1 parent e03a522 commit 960d6af

7 files changed

+172
-28
lines changed

‎.github/workflows/object-storage-adapter-check.yaml‎

Lines changed: 80 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -42,25 +42,57 @@ env:
4242
INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT: '"-Dscalardb.consensus_commit.coordinator.group_commit.enabled=true" "-Dscalardb.consensus_commit.coordinator.group_commit.old_group_abort_timeout_millis=15000" --tests "**.ConsensusCommit**"'
4343
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }}
4444
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
45-
S3_REGION: ap-northeast-1
46-
S3_BUCKET_NAME: scalardb-test-bucket
45+
S3_REGION: us-east-1
46+
S3_BUCKET_BASE_NAME: s3-scalardb-test-bucket
4747
CLOUD_STORAGE_PROJECT_ID: ${{ secrets.CLOUD_STORAGE_PROJECT_ID }}
4848
CLOUD_STORAGE_SERVICE_ACCOUNT_KEY: ${{ secrets.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }}
49-
CLOUD_STORAGE_BUCKET_NAME: scalardb-test-bucket
49+
CLOUD_STORAGE_BUCKET_BASE_NAME: scalardb-test-bucket
5050

5151
jobs:
5252
integration-test-s3:
53-
name: S3 integration test (${{ matrix.mode.label }})
53+
name: S3 integration test (${{ matrix.test_group.label }})
5454
runs-on: ubuntu-latest
5555

5656
strategy:
5757
fail-fast: false
5858
matrix:
59-
mode:
60-
- label: default
59+
test_group:
60+
- label: consensus_commit_default
61+
tests_filter: '--tests "**.ConsensusCommit**"'
62+
bucket_suffix: consensus-commit
6163
group_commit_enabled: false
62-
- label: with_group_commit
64+
- label: consensus_commit_with_group_commit
65+
tests_filter: '--tests "**.ConsensusCommit**"'
66+
bucket_suffix: consensus-commit-gc
6367
group_commit_enabled: true
68+
- label: storage_scan
69+
tests_filter: '--tests "**.ObjectStorageSingle**" --tests "**.ObjectStorageMultiple**"'
70+
bucket_suffix: storage-scan
71+
group_commit_enabled: false
72+
- label: storage_wrapper
73+
tests_filter: '--tests "**.ObjectStorageWrapper**"'
74+
bucket_suffix: storage-wrapper
75+
group_commit_enabled: false
76+
- label: storage_admin
77+
tests_filter: '--tests "**.ObjectStorageAdmin**"'
78+
bucket_suffix: storage-admin
79+
group_commit_enabled: false
80+
- label: storage_cm
81+
tests_filter: '--tests "**.ObjectStorageConditionalMutation**"'
82+
bucket_suffix: storage-cm
83+
group_commit_enabled: false
84+
- label: storage_others
85+
tests_filter: '--tests "**.ObjectStorage**" --tests "!**.ObjectStorageSingle**" --tests "!**.ObjectStorageMultiple**" --tests "!**.ObjectStorageWrapper**" --tests "!**.ObjectStorageAdmin**" --tests "!**.ObjectStorageConditionalMutation**"'
86+
bucket_suffix: storage
87+
group_commit_enabled: false
88+
- label: two_phase_consensus_commit
89+
tests_filter: '--tests "**.TwoPhaseConsensusCommit**"'
90+
bucket_suffix: 2pcc
91+
group_commit_enabled: false
92+
- label: single_crud_operation_transaction
93+
tests_filter: '--tests "**.SingleCrudOperationTransaction**"'
94+
bucket_suffix: single-crud
95+
group_commit_enabled: false
6496

6597
steps:
6698
- uses: actions/checkout@v5
@@ -95,26 +127,59 @@ jobs:
95127
uses: gradle/actions/setup-gradle@v5
96128

97129
- name: Execute Gradle 'integrationTestObjectStorage' task
98-
run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=s3 -Dscalardb.object_storage.endpoint=${{ env.S3_REGION }}/${{ env.S3_BUCKET_NAME }} ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
130+
run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=s3 -Dscalardb.object_storage.endpoint=${{ env.S3_REGION }}/${{ env.S3_BUCKET_BASE_NAME }}-${{ matrix.test_group.bucket_suffix }} -Dscalardb.object_storage.username='${{ env.AWS_ACCESS_KEY_ID }}' -Dscalardb.object_storage.password='${{ env.AWS_SECRET_ACCESS_KEY }}' ${{ matrix.test_group.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} ${{ matrix.test_group.tests_filter }}
99131

100132
- name: Upload Gradle test reports
101133
if: always()
102134
uses: actions/upload-artifact@v5
103135
with:
104-
name: s3_integration_test_reports_${{ matrix.mode.label }}
136+
name: s3_integration_test_reports_${{ matrix.test_group.label }}
105137
path: core/build/reports/tests/integrationTestObjectStorage
138+
106139
integration-test-cloud-storage:
107-
name: Cloud Storage integration test (${{ matrix.mode.label }})
140+
name: Cloud Storage integration test (${{ matrix.test_group.label }})
108141
runs-on: ubuntu-latest
109142

110143
strategy:
111144
fail-fast: false
112145
matrix:
113-
mode:
114-
- label: default
146+
test_group:
147+
- label: consensus_commit_default
148+
tests_filter: '--tests "**.ConsensusCommit**"'
149+
bucket_suffix: consensus-commit
115150
group_commit_enabled: false
116-
- label: with_group_commit
151+
- label: consensus_commit_with_group_commit
152+
tests_filter: '--tests "**.ConsensusCommit**"'
153+
bucket_suffix: consensus-commit-gc
117154
group_commit_enabled: true
155+
- label: storage_scan
156+
tests_filter: '--tests "**.ObjectStorageSingle**" --tests "**.ObjectStorageMultiple**"'
157+
bucket_suffix: storage-scan
158+
group_commit_enabled: false
159+
- label: storage_wrapper
160+
tests_filter: '--tests "**.ObjectStorageWrapper**"'
161+
bucket_suffix: storage-wrapper
162+
group_commit_enabled: false
163+
- label: storage_admin
164+
tests_filter: '--tests "**.ObjectStorageAdmin**"'
165+
bucket_suffix: storage-admin
166+
group_commit_enabled: false
167+
- label: storage_cm
168+
tests_filter: '--tests "**.ObjectStorageConditionalMutation**"'
169+
bucket_suffix: storage-cm
170+
group_commit_enabled: false
171+
- label: storage_others
172+
tests_filter: '--tests "**.ObjectStorage**" --tests "!**.ObjectStorageSingle**" --tests "!**.ObjectStorageMultiple**" --tests "!**.ObjectStorageWrapper**" --tests "!**.ObjectStorageAdmin**" --tests "!**.ObjectStorageConditionalMutation**"'
173+
bucket_suffix: storage
174+
group_commit_enabled: false
175+
- label: two_phase_consensus_commit
176+
tests_filter: '--tests "**.TwoPhaseConsensusCommit**"'
177+
bucket_suffix: 2pcc
178+
group_commit_enabled: false
179+
- label: single_crud_operation_transaction
180+
tests_filter: '--tests "**.SingleCrudOperationTransaction**"'
181+
bucket_suffix: single-crud
182+
group_commit_enabled: false
118183

119184
steps:
120185
- uses: actions/checkout@v5
@@ -149,11 +214,11 @@ jobs:
149214
uses: gradle/actions/setup-gradle@v5
150215

151216
- name: Execute Gradle 'integrationTestObjectStorage' task
152-
run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=cloud-storage -Dscalardb.object_storage.endpoint=scalardb-test-bucket -Dscalardb.object_storage.username=${{ env.CLOUD_STORAGE_PROJECT_ID }} -Dscalardb.object_storage.password=${{ env.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }} ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
217+
run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=cloud-storage -Dscalardb.object_storage.endpoint=${{ env.CLOUD_STORAGE_BUCKET_BASE_NAME }}-${{ matrix.test_group.bucket_suffix }} -Dscalardb.object_storage.username='${{ env.CLOUD_STORAGE_PROJECT_ID }}' -Dscalardb.object_storage.password='${{ env.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }}' ${{ matrix.test_group.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} ${{ matrix.test_group.tests_filter }}
153218

154219
- name: Upload Gradle test reports
155220
if: always()
156221
uses: actions/upload-artifact@v5
157222
with:
158-
name: cloud_storage_integration_test_reports_${{ matrix.mode.label }}
223+
name: cloud_storage_integration_test_reports_${{ matrix.test_group.label }}
159224
path: core/build/reports/tests/integrationTestObjectStorage

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageConditionalMutationIntegrationTest.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,6 @@
66
public class ObjectStorageConditionalMutationIntegrationTest
77
extends DistributedStorageConditionalMutationIntegrationTestBase {
88

9-
@Override
10-
protected int getThreadNum() {
11-
return 3;
12-
}
13-
149
@Override
1510
protected Properties getProperties(String testName) {
1611
return ObjectStorageEnv.getProperties(testName);

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
package com.scalar.db.storage.objectstorage;
22

33
import com.scalar.db.api.DistributedStorageMultipleClusteringKeyScanIntegrationTestBase;
4+
import com.scalar.db.io.Column;
5+
import com.scalar.db.io.DataType;
6+
import com.scalar.db.io.TextColumn;
7+
import com.scalar.db.util.TestUtils;
48
import java.util.Properties;
9+
import java.util.stream.IntStream;
510

611
public class ObjectStorageMultipleClusteringKeyScanIntegrationTest
712
extends DistributedStorageMultipleClusteringKeyScanIntegrationTestBase {
@@ -12,12 +17,18 @@ protected Properties getProperties(String testName) {
1217
}
1318

1419
@Override
15-
protected int getThreadNum() {
16-
return 3;
20+
protected boolean isParallelDdlSupported() {
21+
return false;
1722
}
1823

1924
@Override
20-
protected boolean isParallelDdlSupported() {
21-
return false;
25+
protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType) {
26+
if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) {
27+
// Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value
28+
StringBuilder builder = new StringBuilder();
29+
IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z'));
30+
return TextColumn.of(columnName, builder.toString());
31+
}
32+
return super.getColumnWithMaxValue(columnName, dataType);
2233
}
2334
}

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
package com.scalar.db.storage.objectstorage;
22

33
import com.scalar.db.api.DistributedStorageMultiplePartitionKeyIntegrationTestBase;
4+
import com.scalar.db.io.Column;
5+
import com.scalar.db.io.DataType;
6+
import com.scalar.db.io.TextColumn;
7+
import com.scalar.db.util.TestUtils;
48
import java.util.Properties;
9+
import java.util.stream.IntStream;
510

611
public class ObjectStorageMultiplePartitionKeyIntegrationTest
712
extends DistributedStorageMultiplePartitionKeyIntegrationTestBase {
@@ -12,12 +17,18 @@ protected Properties getProperties(String testName) {
1217
}
1318

1419
@Override
15-
protected int getThreadNum() {
16-
return 3;
20+
protected boolean isParallelDdlSupported() {
21+
return false;
1722
}
1823

1924
@Override
20-
protected boolean isParallelDdlSupported() {
21-
return false;
25+
protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType) {
26+
if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) {
27+
// Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value
28+
StringBuilder builder = new StringBuilder();
29+
IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z'));
30+
return TextColumn.of(columnName, builder.toString());
31+
}
32+
return super.getColumnWithMaxValue(columnName, dataType);
2233
}
2334
}

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSingleClusteringKeyScanIntegrationTest.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,29 @@
11
package com.scalar.db.storage.objectstorage;
22

33
import com.scalar.db.api.DistributedStorageSingleClusteringKeyScanIntegrationTestBase;
4+
import com.scalar.db.io.Column;
5+
import com.scalar.db.io.DataType;
6+
import com.scalar.db.io.TextColumn;
7+
import com.scalar.db.util.TestUtils;
48
import java.util.Properties;
9+
import java.util.stream.IntStream;
510

611
public class ObjectStorageSingleClusteringKeyScanIntegrationTest
712
extends DistributedStorageSingleClusteringKeyScanIntegrationTestBase {
13+
814
@Override
915
protected Properties getProperties(String testName) {
1016
return ObjectStorageEnv.getProperties(testName);
1117
}
18+
19+
@Override
20+
protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType) {
21+
if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) {
22+
// Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value
23+
StringBuilder builder = new StringBuilder();
24+
IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z'));
25+
return TextColumn.of(columnName, builder.toString());
26+
}
27+
return super.getColumnWithMaxValue(columnName, dataType);
28+
}
1229
}

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageSinglePartitionKeyIntegrationTest.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,29 @@
11
package com.scalar.db.storage.objectstorage;
22

33
import com.scalar.db.api.DistributedStorageSinglePartitionKeyIntegrationTestBase;
4+
import com.scalar.db.io.Column;
5+
import com.scalar.db.io.DataType;
6+
import com.scalar.db.io.TextColumn;
7+
import com.scalar.db.util.TestUtils;
48
import java.util.Properties;
9+
import java.util.stream.IntStream;
510

611
public class ObjectStorageSinglePartitionKeyIntegrationTest
712
extends DistributedStorageSinglePartitionKeyIntegrationTestBase {
13+
814
@Override
915
protected Properties getProperties(String testName) {
1016
return ObjectStorageEnv.getProperties(testName);
1117
}
18+
19+
@Override
20+
protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType) {
21+
if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) {
22+
// Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value
23+
StringBuilder builder = new StringBuilder();
24+
IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z'));
25+
return TextColumn.of(columnName, builder.toString());
26+
}
27+
return super.getColumnWithMaxValue(columnName, dataType);
28+
}
1229
}

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionIntegrationTestWithObjectStorage.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,15 @@
11
package com.scalar.db.storage.objectstorage;
22

3+
import com.google.common.util.concurrent.Uninterruptibles;
4+
import com.scalar.db.api.Insert;
5+
import com.scalar.db.api.InsertBuilder;
36
import com.scalar.db.api.TableMetadata;
7+
import com.scalar.db.exception.transaction.TransactionException;
48
import com.scalar.db.io.DataType;
9+
import com.scalar.db.io.Key;
510
import com.scalar.db.transaction.singlecrudoperation.SingleCrudOperationTransactionIntegrationTestBase;
611
import java.util.Properties;
12+
import java.util.concurrent.TimeUnit;
713

814
public class SingleCrudOperationTransactionIntegrationTestWithObjectStorage
915
extends SingleCrudOperationTransactionIntegrationTestBase {
@@ -30,6 +36,28 @@ protected TableMetadata getTableMetadata() {
3036
.build();
3137
}
3238

39+
@Override
40+
protected void populateRecords() throws TransactionException {
41+
for (int i = 0; i < NUM_ACCOUNTS; i++) {
42+
for (int j = 0; j < NUM_TYPES; j++) {
43+
Key partitionKey = Key.ofInt(ACCOUNT_ID, i);
44+
Key clusteringKey = Key.ofInt(ACCOUNT_TYPE, j);
45+
InsertBuilder.Buildable insert =
46+
Insert.newBuilder()
47+
.namespace(namespace)
48+
.table(TABLE)
49+
.partitionKey(partitionKey)
50+
.clusteringKey(clusteringKey);
51+
prepareNonKeyColumns(i, j).forEach(insert::value);
52+
manager.insert(insert.build());
53+
// Sleep to avoid Cloud Storage's request rate limit (1 write per second)
54+
if (ObjectStorageEnv.isCloudStorage()) {
55+
Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
56+
}
57+
}
58+
}
59+
}
60+
3361
@Override
3462
protected Properties getProps(String testName) {
3563
return ObjectStorageEnv.getProperties(testName);

0 commit comments

Comments
 (0)