Skip to content

Commit 17098db

Browse files
ypeckstadtypeckstadt
authored
Merge branch 'master' into feat/data-loader-early-tx-mode-validation
2 parents 947083e + bd39f19 commit 17098db

File tree

34 files changed

+1373
-112
lines changed

34 files changed

+1373
-112
lines changed

‎.github/workflows/object-storage-adapter-check.yaml‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ env:
4444
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
4545
S3_REGION: ap-northeast-1
4646
S3_BUCKET_NAME: scalardb-test-bucket
47+
CLOUD_STORAGE_PROJECT_ID: ${{ secrets.CLOUD_STORAGE_PROJECT_ID }}
48+
CLOUD_STORAGE_SERVICE_ACCOUNT_KEY: ${{ secrets.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }}
49+
CLOUD_STORAGE_BUCKET_NAME: scalardb-test-bucket
4750

4851
jobs:
4952
integration-test-s3:
@@ -98,5 +101,59 @@ jobs:
98101
if: always()
99102
uses: actions/upload-artifact@v5
100103
with:
101-
name: cassandra_3.0_integration_test_reports_${{ matrix.mode.label }}
104+
name: s3_integration_test_reports_${{ matrix.mode.label }}
105+
path: core/build/reports/tests/integrationTestObjectStorage
106+
integration-test-cloud-storage:
107+
name: Cloud Storage integration test (${{ matrix.mode.label }})
108+
runs-on: ubuntu-latest
109+
110+
strategy:
111+
fail-fast: false
112+
matrix:
113+
mode:
114+
- label: default
115+
group_commit_enabled: false
116+
- label: with_group_commit
117+
group_commit_enabled: true
118+
119+
steps:
120+
- uses: actions/checkout@v5
121+
122+
- name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
123+
uses: actions/setup-java@v5
124+
with:
125+
java-version: ${{ env.JAVA_VERSION }}
126+
distribution: ${{ env.JAVA_VENDOR }}
127+
128+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
129+
uses: actions/setup-java@v5
130+
if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
131+
with:
132+
java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
133+
distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
134+
135+
- name: Login to Oracle container registry
136+
uses: docker/login-action@v3
137+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
138+
with:
139+
registry: container-registry.oracle.com
140+
username: ${{ secrets.OCR_USERNAME }}
141+
password: ${{ secrets.OCR_TOKEN }}
142+
143+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
144+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
145+
run: |
146+
container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
147+
docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
148+
- name: Setup Gradle
149+
uses: gradle/actions/setup-gradle@v5
150+
151+
- name: Execute Gradle 'integrationTestObjectStorage' task
152+
run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=cloud-storage -Dscalardb.object_storage.endpoint=scalardb-test-bucket -Dscalardb.object_storage.username=${{ env.CLOUD_STORAGE_PROJECT_ID }} -Dscalardb.object_storage.password=${{ env.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }} ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
153+
154+
- name: Upload Gradle test reports
155+
if: always()
156+
uses: actions/upload-artifact@v5
157+
with:
158+
name: cloud_storage_integration_test_reports_${{ matrix.mode.label }}
102159
path: core/build/reports/tests/integrationTestObjectStorage

‎build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,10 @@ subprojects {
4141
db2DriverVersion = '12.1.3.0'
4242
mariadDbDriverVersion = '3.5.6'
4343
alloyDbJdbcConnectorVersion = '1.2.8'
44+
googleCloudStorageVersion = '2.60.0'
4445
picocliVersion = '4.7.7'
4546
commonsTextVersion = '1.14.0'
47+
caffeineVersion = '2.9.3'
4648
junitVersion = '5.14.1'
4749
commonsLangVersion = '3.20.0'
4850
assertjVersion = '3.27.6'

‎core/build.gradle‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,11 @@ dependencies {
193193
implementation("com.google.cloud:alloydb-jdbc-connector:${alloyDbJdbcConnectorVersion}") {
194194
exclude group: 'org.slf4j', module: 'slf4j-api'
195195
}
196+
implementation("com.google.cloud:google-cloud-storage:${googleCloudStorageVersion}") {
197+
exclude group: 'org.slf4j', module: 'slf4j-api'
198+
}
196199
implementation "org.apache.commons:commons-text:${commonsTextVersion}"
200+
implementation "com.github.ben-manes.caffeine:caffeine:${caffeineVersion}"
197201
testImplementation platform("org.junit:junit-bom:${junitVersion}")
198202
testImplementation 'org.junit.jupiter:junit-jupiter'
199203
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
@@ -226,7 +230,7 @@ task integrationTestCassandra(type: Test) {
226230
classpath = sourceSets.integrationTestCassandra.runtimeClasspath
227231
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
228232
options {
229-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
233+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
230234
}
231235
}
232236

@@ -237,7 +241,7 @@ task integrationTestCosmos(type: Test) {
237241
classpath = sourceSets.integrationTestCosmos.runtimeClasspath
238242
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
239243
options {
240-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
244+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
241245
}
242246
jvmArgs '-XX:MaxDirectMemorySize=4g', '-Xmx6g',
243247
// INFO com.azure.cosmos.implementation.RxDocumentClientImpl - Initializing DocumentClient [3] with serviceEndpoint [https://localhost:8081/], ...
@@ -255,7 +259,7 @@ task integrationTestDynamo(type: Test) {
255259
classpath = sourceSets.integrationTestDynamo.runtimeClasspath
256260
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
257261
options {
258-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
262+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
259263
}
260264
maxParallelForks = 10
261265
}
@@ -267,7 +271,7 @@ task integrationTestJdbc(type: Test) {
267271
classpath = sourceSets.integrationTestJdbc.runtimeClasspath
268272
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
269273
options {
270-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
274+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
271275
}
272276
maxHeapSize = "4g"
273277
}
@@ -279,7 +283,7 @@ task integrationTestObjectStorage(type: Test) {
279283
classpath = sourceSets.integrationTestObjectStorage.runtimeClasspath
280284
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
281285
options {
282-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
286+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
283287
}
284288
}
285289

@@ -290,7 +294,7 @@ task integrationTestMultiStorage(type: Test) {
290294
classpath = sourceSets.integrationTestMultiStorage.runtimeClasspath
291295
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
292296
options {
293-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
297+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
294298
}
295299
}
296300

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageEnv.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.scalar.db.config.DatabaseConfig;
44
import com.scalar.db.storage.objectstorage.blobstorage.BlobStorageConfig;
5+
import com.scalar.db.storage.objectstorage.cloudstorage.CloudStorageConfig;
56
import com.scalar.db.storage.objectstorage.s3.S3Config;
67
import java.util.Collections;
78
import java.util.Map;
@@ -76,6 +77,11 @@ public static boolean isBlobStorage() {
7677
.equals(BlobStorageConfig.STORAGE_NAME);
7778
}
7879

80+
public static boolean isCloudStorage() {
81+
return System.getProperty(PROP_OBJECT_STORAGE_STORAGE, DEFAULT_OBJECT_STORAGE_STORAGE)
82+
.equals(CloudStorageConfig.STORAGE_NAME);
83+
}
84+
7985
public static boolean isS3() {
8086
return System.getProperty(PROP_OBJECT_STORAGE_STORAGE, DEFAULT_OBJECT_STORAGE_STORAGE)
8187
.equals(S3Config.STORAGE_NAME);

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageWrapperIntegrationTest.java‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,11 @@ public class ObjectStorageWrapperIntegrationTest {
2828
private static final String TEST_OBJECT2 = "test-object2";
2929
private static final String TEST_OBJECT3 = "test-object3";
3030
private static final int BLOB_STORAGE_LIST_MAX_KEYS = 5000;
31+
private static final int CLOUD_STORAGE_LIST_MAX_KEYS = 1000;
32+
private static final int S3_LIST_MAX_KEYS = 1000;
3133

3234
private ObjectStorageWrapper wrapper;
35+
private int listMaxKeys;
3336

3437
@BeforeAll
3538
public void beforeAll() throws ObjectStorageWrapperException {
@@ -38,6 +41,16 @@ public void beforeAll() throws ObjectStorageWrapperException {
3841
ObjectStorageUtils.getObjectStorageConfig(new DatabaseConfig(properties));
3942
wrapper = ObjectStorageWrapperFactory.create(objectStorageConfig);
4043
createObjects();
44+
45+
if (ObjectStorageEnv.isBlobStorage()) {
46+
listMaxKeys = BLOB_STORAGE_LIST_MAX_KEYS;
47+
} else if (ObjectStorageEnv.isCloudStorage()) {
48+
listMaxKeys = CLOUD_STORAGE_LIST_MAX_KEYS;
49+
} else if (ObjectStorageEnv.isS3()) {
50+
listMaxKeys = S3_LIST_MAX_KEYS;
51+
} else {
52+
throw new AssertionError();
53+
}
4154
}
4255

4356
@AfterAll
@@ -152,14 +165,14 @@ public void update_NonExistingObjectKeyGiven_ShouldThrowPreconditionFailedExcept
152165
String objectKey = "non-existing-key";
153166

154167
// Act Assert
155-
assertThatCode(() -> wrapper.update(objectKey, "some-object", "some-version"))
168+
assertThatCode(() -> wrapper.update(objectKey, "some-object", "123456789"))
156169
.isInstanceOf(PreconditionFailedException.class);
157170
}
158171

159172
@Test
160173
public void update_WrongVersionGiven_ShouldThrowPreconditionFailedException() {
161174
// Arrange
162-
String wrongVersion = "wrong-version";
175+
String wrongVersion = "123456789";
163176

164177
// Act Assert
165178
assertThatCode(() -> wrapper.update(TEST_KEY2, "another-object", wrongVersion))
@@ -219,7 +232,7 @@ public void delete_ExistingObjectKeyWithWrongVersionGiven_ShouldThrowPreconditio
219232
// Arrange
220233
Optional<ObjectStorageWrapperResponse> response1 = wrapper.get(TEST_KEY1);
221234
assertThat(response1.isPresent()).isTrue();
222-
String wrongVersion = "wrong-version";
235+
String wrongVersion = "123456789";
223236

224237
// Act Assert
225238
assertThatCode(() -> wrapper.delete(TEST_KEY1, wrongVersion))
@@ -253,7 +266,7 @@ public void getKeys_WithNonExistingPrefix_ShouldReturnEmptySet() throws Exceptio
253266
public void getKeys_WithPrefixForTheNumberOfObjectsExceedingTheListLimit_ShouldReturnAllKeys()
254267
throws Exception {
255268
String prefix = "prefix-";
256-
int numberOfObjects = BLOB_STORAGE_LIST_MAX_KEYS + 1;
269+
int numberOfObjects = listMaxKeys + 1;
257270
try {
258271
// Arrange
259272
for (int i = 0; i < numberOfObjects; i++) {
@@ -313,7 +326,7 @@ public void deleteByPrefix_WithNonExistingPrefix_ShouldDoNothing() throws Except
313326
deleteByPrefix_WithPrefixForTheNumberOfObjectsExceedingTheListLimit_ShouldDeleteAllObjects()
314327
throws Exception {
315328
String prefix = "prefix-";
316-
int numberOfObjects = BLOB_STORAGE_LIST_MAX_KEYS + 1;
329+
int numberOfObjects = listMaxKeys + 1;
317330
try {
318331
// Arrange
319332
for (int i = 0; i < numberOfObjects; i++) {

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageWrapperLargeObjectWriteIntegrationTest.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import com.scalar.db.config.DatabaseConfig;
77
import com.scalar.db.storage.objectstorage.blobstorage.BlobStorageConfig;
8+
import com.scalar.db.storage.objectstorage.cloudstorage.CloudStorageConfig;
89
import com.scalar.db.storage.objectstorage.s3.S3Config;
910
import java.util.Arrays;
1011
import java.util.Optional;
@@ -47,6 +48,13 @@ public void beforeAll() throws ObjectStorageWrapperException {
4748
BlobStorageConfig.PARALLEL_UPLOAD_THRESHOLD_IN_BYTES,
4849
String.valueOf(parallelUploadUnit * 2));
4950
parallelUploadThresholdInBytes = parallelUploadUnit * 2;
51+
} else if (ObjectStorageEnv.isCloudStorage()) {
52+
// Minimum block size must be greater than or equal to 256KB for Cloud Storage
53+
Long parallelUploadUnit = 256 * 1024L; // 256KB
54+
properties.setProperty(
55+
CloudStorageConfig.PARALLEL_UPLOAD_BLOCK_SIZE_IN_BYTES,
56+
String.valueOf(parallelUploadUnit));
57+
parallelUploadThresholdInBytes = parallelUploadUnit * 2;
5058
} else if (ObjectStorageEnv.isS3()) {
5159
// Minimum part size must be greater than or equal to 5MB for S3
5260
Long parallelUploadUnit = 5 * 1024 * 1024L; // 5MB
@@ -59,7 +67,7 @@ public void beforeAll() throws ObjectStorageWrapperException {
5967
throw new AssertionError();
6068
}
6169

62-
char[] charArray = new char[(int) parallelUploadThresholdInBytes];
70+
char[] charArray = new char[(int) parallelUploadThresholdInBytes + 1];
6371
Arrays.fill(charArray, 'a');
6472
testObject1 = new String(charArray);
6573
Arrays.fill(charArray, 'b');
@@ -163,14 +171,14 @@ public void update_NonExistingObjectKeyGiven_ShouldThrowPreconditionFailedExcept
163171
String objectKey = "non-existing-key";
164172

165173
// Act Assert
166-
assertThatCode(() -> wrapper.update(objectKey, "some-object", "some-version"))
174+
assertThatCode(() -> wrapper.update(objectKey, "some-object", "123456789"))
167175
.isInstanceOf(PreconditionFailedException.class);
168176
}
169177

170178
@Test
171179
public void update_WrongVersionGiven_ShouldThrowPreconditionFailedException() {
172180
// Arrange
173-
String wrongVersion = "wrong-version";
181+
String wrongVersion = "123456789";
174182

175183
// Act Assert
176184
assertThatCode(() -> wrapper.update(TEST_KEY2, "another-object", wrongVersion))

‎core/src/main/java/com/scalar/db/common/CoreError.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -931,6 +931,18 @@ public enum CoreError implements ScalarDbError {
931931
"Conditions on indexed columns in cross-partition scan operations are not allowed in the SERIALIZABLE isolation level",
932932
"",
933933
""),
934+
OBJECT_STORAGE_CLOUD_STORAGE_SERVICE_ACCOUNT_KEY_NOT_FOUND(
935+
Category.USER_ERROR,
936+
"0263",
937+
"The service account key for Cloud Storage was not found.",
938+
"",
939+
""),
940+
OBJECT_STORAGE_CLOUD_STORAGE_SERVICE_ACCOUNT_KEY_LOAD_FAILED(
941+
Category.USER_ERROR,
942+
"0264",
943+
"Failed to load the service account key for Cloud Storage.",
944+
"",
945+
""),
934946

935947
//
936948
// Errors for the concurrency error category

‎core/src/main/java/com/scalar/db/common/TableMetadataManager.java‎

Lines changed: 14 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,37 @@
11
package com.scalar.db.common;
22

3-
import com.google.common.cache.CacheBuilder;
4-
import com.google.common.cache.CacheLoader;
5-
import com.google.common.cache.LoadingCache;
3+
import com.github.benmanes.caffeine.cache.Caffeine;
4+
import com.github.benmanes.caffeine.cache.LoadingCache;
65
import com.scalar.db.api.Admin;
76
import com.scalar.db.api.Operation;
87
import com.scalar.db.api.TableMetadata;
98
import com.scalar.db.exception.storage.ExecutionException;
109
import com.scalar.db.util.ScalarDbUtils;
1110
import com.scalar.db.util.ThrowableFunction;
1211
import java.util.Objects;
13-
import java.util.Optional;
12+
import java.util.concurrent.CompletionException;
1413
import java.util.concurrent.TimeUnit;
15-
import javax.annotation.Nonnull;
14+
import javax.annotation.Nullable;
1615
import javax.annotation.concurrent.ThreadSafe;
1716

1817
/** A class that manages and caches table metadata */
1918
@ThreadSafe
2019
public class TableMetadataManager {
2120

22-
private final LoadingCache<TableKey, Optional<TableMetadata>> tableMetadataCache;
21+
private final LoadingCache<TableKey, TableMetadata> tableMetadataCache;
2322

2423
public TableMetadataManager(Admin admin, long cacheExpirationTimeSecs) {
25-
this(
26-
key -> Optional.ofNullable(admin.getTableMetadata(key.namespace, key.table)),
27-
cacheExpirationTimeSecs);
24+
this(key -> admin.getTableMetadata(key.namespace, key.table), cacheExpirationTimeSecs);
2825
}
2926

3027
public TableMetadataManager(
31-
ThrowableFunction<TableKey, Optional<TableMetadata>, Exception> getTableMetadataFunc,
28+
ThrowableFunction<TableKey, TableMetadata, Exception> getTableMetadataFunc,
3229
long cacheExpirationTimeSecs) {
33-
CacheBuilder<Object, Object> builder = CacheBuilder.newBuilder();
30+
Caffeine<Object, Object> builder = Caffeine.newBuilder();
3431
if (cacheExpirationTimeSecs >= 0) {
3532
builder.expireAfterWrite(cacheExpirationTimeSecs, TimeUnit.SECONDS);
3633
}
37-
tableMetadataCache =
38-
builder.build(
39-
new CacheLoader<TableKey, Optional<TableMetadata>>() {
40-
@Nonnull
41-
@Override
42-
public Optional<TableMetadata> load(@Nonnull TableKey key) throws Exception {
43-
return getTableMetadataFunc.apply(key);
44-
}
45-
});
34+
tableMetadataCache = builder.build(getTableMetadataFunc::apply);
4635
}
4736

4837
/**
@@ -61,22 +50,23 @@ public TableMetadata getTableMetadata(Operation operation) throws ExecutionExcep
6150
}
6251

6352
/**
64-
* Returns a table metadata corresponding to the specified operation.
53+
* Returns a table metadata corresponding to the specified namespace and table.
6554
*
6655
* @param namespace a namespace to retrieve
6756
* @param table a table to retrieve
6857
* @return a table metadata. null if the table is not found.
6958
* @throws ExecutionException if the operation fails
7059
*/
60+
@Nullable
7161
public TableMetadata getTableMetadata(String namespace, String table) throws ExecutionException {
7262
try {
7363
TableKey key = new TableKey(namespace, table);
74-
return tableMetadataCache.get(key).orElse(null);
75-
} catch (java.util.concurrent.ExecutionException e) {
64+
return tableMetadataCache.get(key);
65+
} catch (CompletionException e) {
7666
throw new ExecutionException(
7767
CoreError.GETTING_TABLE_METADATA_FAILED.buildMessage(
7868
ScalarDbUtils.getFullTableName(namespace, table)),
79-
e);
69+
e.getCause());
8070
}
8171
}
8272

0 commit comments

Comments
 (0)