Skip to content

Commit 364d34e

Browse files
brfrn169brfrn169
committed
Merge branch 'master' into introduce-virtual-tables-in-storage-abstraction
2 parents d5693a4 + 4cfac72 commit 364d34e

File tree

67 files changed

+1870
-568
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+1870
-568
lines changed

‎.github/workflows/object-storage-adapter-check.yaml‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ env:
4444
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
4545
S3_REGION: ap-northeast-1
4646
S3_BUCKET_NAME: scalardb-test-bucket
47+
CLOUD_STORAGE_PROJECT_ID: ${{ secrets.CLOUD_STORAGE_PROJECT_ID }}
48+
CLOUD_STORAGE_SERVICE_ACCOUNT_KEY: ${{ secrets.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }}
49+
CLOUD_STORAGE_BUCKET_NAME: scalardb-test-bucket
4750

4851
jobs:
4952
integration-test-s3:
@@ -98,5 +101,59 @@ jobs:
98101
if: always()
99102
uses: actions/upload-artifact@v5
100103
with:
101-
name: cassandra_3.0_integration_test_reports_${{ matrix.mode.label }}
104+
name: s3_integration_test_reports_${{ matrix.mode.label }}
105+
path: core/build/reports/tests/integrationTestObjectStorage
106+
integration-test-cloud-storage:
107+
name: Cloud Storage integration test (${{ matrix.mode.label }})
108+
runs-on: ubuntu-latest
109+
110+
strategy:
111+
fail-fast: false
112+
matrix:
113+
mode:
114+
- label: default
115+
group_commit_enabled: false
116+
- label: with_group_commit
117+
group_commit_enabled: true
118+
119+
steps:
120+
- uses: actions/checkout@v5
121+
122+
- name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
123+
uses: actions/setup-java@v5
124+
with:
125+
java-version: ${{ env.JAVA_VERSION }}
126+
distribution: ${{ env.JAVA_VENDOR }}
127+
128+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
129+
uses: actions/setup-java@v5
130+
if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
131+
with:
132+
java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
133+
distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
134+
135+
- name: Login to Oracle container registry
136+
uses: docker/login-action@v3
137+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
138+
with:
139+
registry: container-registry.oracle.com
140+
username: ${{ secrets.OCR_USERNAME }}
141+
password: ${{ secrets.OCR_TOKEN }}
142+
143+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
144+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
145+
run: |
146+
container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
147+
docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
148+
- name: Setup Gradle
149+
uses: gradle/actions/setup-gradle@v5
150+
151+
- name: Execute Gradle 'integrationTestObjectStorage' task
152+
run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=cloud-storage -Dscalardb.object_storage.endpoint=scalardb-test-bucket -Dscalardb.object_storage.username=${{ env.CLOUD_STORAGE_PROJECT_ID }} -Dscalardb.object_storage.password=${{ env.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }} ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
153+
154+
- name: Upload Gradle test reports
155+
if: always()
156+
uses: actions/upload-artifact@v5
157+
with:
158+
name: cloud_storage_integration_test_reports_${{ matrix.mode.label }}
102159
path: core/build/reports/tests/integrationTestObjectStorage

‎build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ subprojects {
4141
db2DriverVersion = '12.1.3.0'
4242
mariadDbDriverVersion = '3.5.6'
4343
alloyDbJdbcConnectorVersion = '1.2.8'
44+
googleCloudStorageVersion = '2.60.0'
4445
picocliVersion = '4.7.7'
4546
commonsTextVersion = '1.14.0'
4647
caffeineVersion = '2.9.3'

‎core/build.gradle‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,9 @@ dependencies {
193193
implementation("com.google.cloud:alloydb-jdbc-connector:${alloyDbJdbcConnectorVersion}") {
194194
exclude group: 'org.slf4j', module: 'slf4j-api'
195195
}
196+
implementation("com.google.cloud:google-cloud-storage:${googleCloudStorageVersion}") {
197+
exclude group: 'org.slf4j', module: 'slf4j-api'
198+
}
196199
implementation "org.apache.commons:commons-text:${commonsTextVersion}"
197200
implementation "com.github.ben-manes.caffeine:caffeine:${caffeineVersion}"
198201
testImplementation platform("org.junit:junit-bom:${junitVersion}")
@@ -227,7 +230,7 @@ task integrationTestCassandra(type: Test) {
227230
classpath = sourceSets.integrationTestCassandra.runtimeClasspath
228231
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
229232
options {
230-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
233+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
231234
}
232235
}
233236

@@ -238,7 +241,7 @@ task integrationTestCosmos(type: Test) {
238241
classpath = sourceSets.integrationTestCosmos.runtimeClasspath
239242
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
240243
options {
241-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
244+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
242245
}
243246
jvmArgs '-XX:MaxDirectMemorySize=4g', '-Xmx6g',
244247
// INFO com.azure.cosmos.implementation.RxDocumentClientImpl - Initializing DocumentClient [3] with serviceEndpoint [https://localhost:8081/], ...
@@ -256,7 +259,7 @@ task integrationTestDynamo(type: Test) {
256259
classpath = sourceSets.integrationTestDynamo.runtimeClasspath
257260
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
258261
options {
259-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
262+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
260263
}
261264
maxParallelForks = 10
262265
}
@@ -268,7 +271,7 @@ task integrationTestJdbc(type: Test) {
268271
classpath = sourceSets.integrationTestJdbc.runtimeClasspath
269272
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
270273
options {
271-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
274+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
272275
}
273276
maxHeapSize = "4g"
274277
}
@@ -280,7 +283,7 @@ task integrationTestObjectStorage(type: Test) {
280283
classpath = sourceSets.integrationTestObjectStorage.runtimeClasspath
281284
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
282285
options {
283-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
286+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
284287
}
285288
}
286289

@@ -291,7 +294,7 @@ task integrationTestMultiStorage(type: Test) {
291294
classpath = sourceSets.integrationTestMultiStorage.runtimeClasspath
292295
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
293296
options {
294-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
297+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
295298
}
296299
}
297300

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageEnv.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.scalar.db.config.DatabaseConfig;
44
import com.scalar.db.storage.objectstorage.blobstorage.BlobStorageConfig;
5+
import com.scalar.db.storage.objectstorage.cloudstorage.CloudStorageConfig;
56
import com.scalar.db.storage.objectstorage.s3.S3Config;
67
import java.util.Collections;
78
import java.util.Map;
@@ -76,6 +77,11 @@ public static boolean isBlobStorage() {
7677
.equals(BlobStorageConfig.STORAGE_NAME);
7778
}
7879

80+
public static boolean isCloudStorage() {
81+
return System.getProperty(PROP_OBJECT_STORAGE_STORAGE, DEFAULT_OBJECT_STORAGE_STORAGE)
82+
.equals(CloudStorageConfig.STORAGE_NAME);
83+
}
84+
7985
public static boolean isS3() {
8086
return System.getProperty(PROP_OBJECT_STORAGE_STORAGE, DEFAULT_OBJECT_STORAGE_STORAGE)
8187
.equals(S3Config.STORAGE_NAME);

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageWrapperIntegrationTest.java‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,11 @@ public class ObjectStorageWrapperIntegrationTest {
2828
private static final String TEST_OBJECT2 = "test-object2";
2929
private static final String TEST_OBJECT3 = "test-object3";
3030
private static final int BLOB_STORAGE_LIST_MAX_KEYS = 5000;
31+
private static final int CLOUD_STORAGE_LIST_MAX_KEYS = 1000;
32+
private static final int S3_LIST_MAX_KEYS = 1000;
3133

3234
private ObjectStorageWrapper wrapper;
35+
private int listMaxKeys;
3336

3437
@BeforeAll
3538
public void beforeAll() throws ObjectStorageWrapperException {
@@ -38,6 +41,16 @@ public void beforeAll() throws ObjectStorageWrapperException {
3841
ObjectStorageUtils.getObjectStorageConfig(new DatabaseConfig(properties));
3942
wrapper = ObjectStorageWrapperFactory.create(objectStorageConfig);
4043
createObjects();
44+
45+
if (ObjectStorageEnv.isBlobStorage()) {
46+
listMaxKeys = BLOB_STORAGE_LIST_MAX_KEYS;
47+
} else if (ObjectStorageEnv.isCloudStorage()) {
48+
listMaxKeys = CLOUD_STORAGE_LIST_MAX_KEYS;
49+
} else if (ObjectStorageEnv.isS3()) {
50+
listMaxKeys = S3_LIST_MAX_KEYS;
51+
} else {
52+
throw new AssertionError();
53+
}
4154
}
4255

4356
@AfterAll
@@ -152,14 +165,14 @@ public void update_NonExistingObjectKeyGiven_ShouldThrowPreconditionFailedExcept
152165
String objectKey = "non-existing-key";
153166

154167
// Act Assert
155-
assertThatCode(() -> wrapper.update(objectKey, "some-object", "some-version"))
168+
assertThatCode(() -> wrapper.update(objectKey, "some-object", "123456789"))
156169
.isInstanceOf(PreconditionFailedException.class);
157170
}
158171

159172
@Test
160173
public void update_WrongVersionGiven_ShouldThrowPreconditionFailedException() {
161174
// Arrange
162-
String wrongVersion = "wrong-version";
175+
String wrongVersion = "123456789";
163176

164177
// Act Assert
165178
assertThatCode(() -> wrapper.update(TEST_KEY2, "another-object", wrongVersion))
@@ -219,7 +232,7 @@ public void delete_ExistingObjectKeyWithWrongVersionGiven_ShouldThrowPreconditio
219232
// Arrange
220233
Optional<ObjectStorageWrapperResponse> response1 = wrapper.get(TEST_KEY1);
221234
assertThat(response1.isPresent()).isTrue();
222-
String wrongVersion = "wrong-version";
235+
String wrongVersion = "123456789";
223236

224237
// Act Assert
225238
assertThatCode(() -> wrapper.delete(TEST_KEY1, wrongVersion))
@@ -253,7 +266,7 @@ public void getKeys_WithNonExistingPrefix_ShouldReturnEmptySet() throws Exceptio
253266
public void getKeys_WithPrefixForTheNumberOfObjectsExceedingTheListLimit_ShouldReturnAllKeys()
254267
throws Exception {
255268
String prefix = "prefix-";
256-
int numberOfObjects = BLOB_STORAGE_LIST_MAX_KEYS + 1;
269+
int numberOfObjects = listMaxKeys + 1;
257270
try {
258271
// Arrange
259272
for (int i = 0; i < numberOfObjects; i++) {
@@ -313,7 +326,7 @@ public void deleteByPrefix_WithNonExistingPrefix_ShouldDoNothing() throws Except
313326
deleteByPrefix_WithPrefixForTheNumberOfObjectsExceedingTheListLimit_ShouldDeleteAllObjects()
314327
throws Exception {
315328
String prefix = "prefix-";
316-
int numberOfObjects = BLOB_STORAGE_LIST_MAX_KEYS + 1;
329+
int numberOfObjects = listMaxKeys + 1;
317330
try {
318331
// Arrange
319332
for (int i = 0; i < numberOfObjects; i++) {

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageWrapperLargeObjectWriteIntegrationTest.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import com.scalar.db.config.DatabaseConfig;
77
import com.scalar.db.storage.objectstorage.blobstorage.BlobStorageConfig;
8+
import com.scalar.db.storage.objectstorage.cloudstorage.CloudStorageConfig;
89
import com.scalar.db.storage.objectstorage.s3.S3Config;
910
import java.util.Arrays;
1011
import java.util.Optional;
@@ -47,6 +48,13 @@ public void beforeAll() throws ObjectStorageWrapperException {
4748
BlobStorageConfig.PARALLEL_UPLOAD_THRESHOLD_IN_BYTES,
4849
String.valueOf(parallelUploadUnit * 2));
4950
parallelUploadThresholdInBytes = parallelUploadUnit * 2;
51+
} else if (ObjectStorageEnv.isCloudStorage()) {
52+
// Minimum block size must be greater than or equal to 256KB for Cloud Storage
53+
Long parallelUploadUnit = 256 * 1024L; // 256KB
54+
properties.setProperty(
55+
CloudStorageConfig.PARALLEL_UPLOAD_BLOCK_SIZE_IN_BYTES,
56+
String.valueOf(parallelUploadUnit));
57+
parallelUploadThresholdInBytes = parallelUploadUnit * 2;
5058
} else if (ObjectStorageEnv.isS3()) {
5159
// Minimum part size must be greater than or equal to 5MB for S3
5260
Long parallelUploadUnit = 5 * 1024 * 1024L; // 5MB
@@ -59,7 +67,7 @@ public void beforeAll() throws ObjectStorageWrapperException {
5967
throw new AssertionError();
6068
}
6169

62-
char[] charArray = new char[(int) parallelUploadThresholdInBytes];
70+
char[] charArray = new char[(int) parallelUploadThresholdInBytes + 1];
6371
Arrays.fill(charArray, 'a');
6472
testObject1 = new String(charArray);
6573
Arrays.fill(charArray, 'b');
@@ -163,14 +171,14 @@ public void update_NonExistingObjectKeyGiven_ShouldThrowPreconditionFailedExcept
163171
String objectKey = "non-existing-key";
164172

165173
// Act Assert
166-
assertThatCode(() -> wrapper.update(objectKey, "some-object", "some-version"))
174+
assertThatCode(() -> wrapper.update(objectKey, "some-object", "123456789"))
167175
.isInstanceOf(PreconditionFailedException.class);
168176
}
169177

170178
@Test
171179
public void update_WrongVersionGiven_ShouldThrowPreconditionFailedException() {
172180
// Arrange
173-
String wrongVersion = "wrong-version";
181+
String wrongVersion = "123456789";
174182

175183
// Act Assert
176184
assertThatCode(() -> wrapper.update(TEST_KEY2, "another-object", wrongVersion))

‎core/src/main/java/com/scalar/db/common/CoreError.java‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -931,69 +931,81 @@ public enum CoreError implements ScalarDbError {
931931
"Conditions on indexed columns in cross-partition scan operations are not allowed in the SERIALIZABLE isolation level",
932932
"",
933933
""),
934-
VIRTUAL_TABLE_NOT_SUPPORTED_IN_STORAGE(
934+
OBJECT_STORAGE_CLOUD_STORAGE_SERVICE_ACCOUNT_KEY_NOT_FOUND(
935935
Category.USER_ERROR,
936936
"0263",
937+
"The service account key for Cloud Storage was not found.",
938+
"",
939+
""),
940+
OBJECT_STORAGE_CLOUD_STORAGE_SERVICE_ACCOUNT_KEY_LOAD_FAILED(
941+
Category.USER_ERROR,
942+
"0264",
943+
"Failed to load the service account key for Cloud Storage.",
944+
"",
945+
""),
946+
VIRTUAL_TABLE_NOT_SUPPORTED_IN_STORAGE(
947+
Category.USER_ERROR,
948+
"0265",
937949
"To support virtual tables, the atomicity unit of the storage must be at least at the namespace level. Storage: %s; Atomicity unit: %s",
938950
"",
939951
""),
940952
VIRTUAL_TABLE_SOURCE_TABLES_OUTSIDE_OF_ATOMICITY_UNIT(
941953
Category.USER_ERROR,
942-
"0264",
954+
"0266",
943955
"The source tables must reside within the atomicity unit of the storage. Storage: %s; Atomicity unit: %s; Left source table: %s; Right source table: %s",
944956
"",
945957
""),
946958
DYNAMO_VIRTUAL_TABLE_NOT_SUPPORTED(
947959
Category.USER_ERROR,
948-
"0265",
960+
"0267",
949961
"The virtual table functionality is not supported in DynamoDB",
950962
"",
951963
""),
952964
VIRTUAL_TABLE_SOURCE_TABLES_HAVE_DIFFERENT_PRIMARY_KEY(
953965
Category.USER_ERROR,
954-
"0266",
966+
"0268",
955967
"The source tables must have the same primary key. Left source table: %s; Right source table: %s",
956968
"",
957969
""),
958970
VIRTUAL_TABLE_SOURCE_TABLES_HAVE_DIFFERENT_PRIMARY_KEY_TYPES(
959971
Category.USER_ERROR,
960-
"0267",
972+
"0269",
961973
"The source tables must have the same data types for primary key column. Column: %s; Left source table: %s; Right source table: %s",
962974
"",
963975
""),
964976
VIRTUAL_TABLE_SOURCE_TABLES_HAVE_DIFFERENT_CLUSTERING_ORDERS(
965977
Category.USER_ERROR,
966-
"0268",
978+
"0270",
967979
"The source tables must have the same clustering orders for clustering key column. Column: %s; Left source table: %s; Right source table: %s",
968980
"",
969981
""),
970982
VIRTUAL_TABLE_SOURCE_TABLES_HAVE_CONFLICTING_COLUMN_NAMES(
971983
Category.USER_ERROR,
972-
"0269",
984+
"0271",
973985
"The source tables have conflicting non-key column names. Left source table: %s; Right source table: %s; Conflicting columns: %s",
974986
"",
975987
""),
976988
VIRTUAL_TABLE_CANNOT_USE_VIRTUAL_TABLE_AS_SOURCE(
977989
Category.USER_ERROR,
978-
"0270",
990+
"0272",
979991
"Virtual tables cannot be used as source tables. Source table: %s",
980992
"",
981993
""),
982994
VIRTUAL_TABLE_SOURCE_TABLES_IN_DIFFERENT_STORAGES(
983995
Category.USER_ERROR,
984-
"0271",
996+
"0273",
985997
"The source tables must be in the same storage. Left source table: %s; Right source table: %s",
986998
"",
987999
""),
9881000
VIRTUAL_TABLE_IN_DIFFERENT_STORAGE_FROM_SOURCE_TABLES(
9891001
Category.USER_ERROR,
990-
"0272",
1002+
"0274",
9911003
"The virtual table must be in the same storage as its source tables. Virtual table: %s; Left source table: %s; Right source table: %s",
9921004
"",
9931005
""),
9941006
SOURCE_TABLES_CANNOT_BE_DROPPED_WHILE_VIRTUAL_TABLES_EXIST(
9951007
Category.USER_ERROR,
996-
"0273",
1008+
"0275",
9971009
"Source tables cannot be dropped while virtual tables depending on them exist. Source table: %s; Virtual tables: %s",
9981010
"",
9991011
""),

0 commit comments

Comments
 (0)