Skip to content

Commit e4fc125

Browse files
committed
Fix based on review
1 parent 2ef8e05 commit e4fc125

File tree

4 files changed

+53
-38
lines changed

4 files changed

+53
-38
lines changed

core/src/main/java/com/scalar/db/common/CoreError.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -931,6 +931,18 @@ public enum CoreError implements ScalarDbError {
931931
"Conditions on indexed columns in cross-partition scan operations are not allowed in the SERIALIZABLE isolation level",
932932
"",
933933
""),
934+
OBJECT_STORAGE_CLOUD_STORAGE_SERVICE_ACCOUNT_KEY_NOT_FOUND(
935+
Category.USER_ERROR,
936+
"0263",
937+
"The service account key for Cloud Storage is not found.",
938+
"",
939+
""),
940+
OBJECT_STORAGE_CLOUD_STORAGE_SERVICE_ACCOUNT_KEY_LOAD_FAILED(
941+
Category.USER_ERROR,
942+
"0264",
943+
"Failed to load the service account key for Cloud Storage.",
944+
"",
945+
""),
934946

935947
//
936948
// Errors for the concurrency error category

core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageWrapper.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public interface ObjectStorageWrapper {
6868
/**
6969
* Delete objects with the specified prefix from the storage. <br>
7070
* <br>
71-
* <strong>Attention:</strong> This method does not guarantee atomicity and assume to be used
71+
* <strong>Attention:</strong> This method does not guarantee atomicity and is assumed to be used
7272
* where concurrent operations do not occur.
7373
*
7474
* @param prefix the prefix of the objects to delete

core/src/main/java/com/scalar/db/storage/objectstorage/cloudstorage/CloudStorageConfig.java

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,14 @@
22

33
import static com.scalar.db.config.ConfigUtils.getInt;
44

5+
import com.google.auth.Credentials;
6+
import com.google.auth.oauth2.ServiceAccountCredentials;
7+
import com.scalar.db.common.CoreError;
58
import com.scalar.db.config.DatabaseConfig;
69
import com.scalar.db.storage.objectstorage.ObjectStorageConfig;
10+
import java.io.ByteArrayInputStream;
11+
import java.io.IOException;
12+
import java.nio.charset.StandardCharsets;
713
import java.util.Optional;
814
import org.slf4j.Logger;
915
import org.slf4j.LoggerFactory;
@@ -71,6 +77,21 @@ public String getProjectId() {
7177
return projectId;
7278
}
7379

80+
public Credentials getCredentials() {
81+
String serviceAccountJson = getPassword();
82+
if (serviceAccountJson == null) {
83+
throw new IllegalArgumentException(
84+
CoreError.OBJECT_STORAGE_CLOUD_STORAGE_SERVICE_ACCOUNT_KEY_NOT_FOUND.buildMessage());
85+
}
86+
try (ByteArrayInputStream keyStream =
87+
new ByteArrayInputStream(serviceAccountJson.getBytes(StandardCharsets.UTF_8))) {
88+
return ServiceAccountCredentials.fromStream(keyStream);
89+
} catch (IOException e) {
90+
throw new IllegalArgumentException(
91+
CoreError.OBJECT_STORAGE_CLOUD_STORAGE_SERVICE_ACCOUNT_KEY_LOAD_FAILED.buildMessage());
92+
}
93+
}
94+
7495
public Optional<Integer> getParallelUploadBlockSizeInBytes() {
7596
return Optional.ofNullable(parallelUploadBlockSizeInBytes);
7697
}

core/src/main/java/com/scalar/db/storage/objectstorage/cloudstorage/CloudStorageWrapper.java

Lines changed: 19 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package com.scalar.db.storage.objectstorage.cloudstorage;
22

33
import com.google.api.gax.paging.Page;
4-
import com.google.auth.oauth2.ServiceAccountCredentials;
54
import com.google.cloud.WriteChannel;
65
import com.google.cloud.storage.Blob;
76
import com.google.cloud.storage.BlobId;
@@ -16,7 +15,6 @@
1615
import com.scalar.db.storage.objectstorage.ObjectStorageWrapperResponse;
1716
import com.scalar.db.storage.objectstorage.PreconditionFailedException;
1817
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
19-
import java.io.ByteArrayInputStream;
2018
import java.io.IOException;
2119
import java.nio.ByteBuffer;
2220
import java.nio.charset.StandardCharsets;
@@ -36,22 +34,10 @@ public class CloudStorageWrapper implements ObjectStorageWrapper {
3634
private final Integer parallelUploadBlockSizeInBytes;
3735

3836
public CloudStorageWrapper(CloudStorageConfig config) {
39-
ServiceAccountCredentials credentials;
40-
if (config.getPassword() == null) {
41-
throw new RuntimeException(
42-
"Service account credentials are not provided in the password field");
43-
}
44-
try (ByteArrayInputStream keyStream =
45-
new ByteArrayInputStream(config.getPassword().getBytes(StandardCharsets.UTF_8))) {
46-
credentials = ServiceAccountCredentials.fromStream(keyStream);
47-
} catch (IOException e) {
48-
throw new RuntimeException("Failed to load the service account credentials", e);
49-
}
50-
5137
storage =
5238
StorageOptions.newBuilder()
5339
.setProjectId(config.getProjectId())
54-
.setCredentials(credentials)
40+
.setCredentials(config.getCredentials())
5541
.build()
5642
.getService();
5743
bucket = config.getBucket();
@@ -108,18 +94,8 @@ public Set<String> getKeys(String prefix) throws ObjectStorageWrapperException {
10894
@Override
10995
public void insert(String key, String object) throws ObjectStorageWrapperException {
11096
try {
111-
byte[] data = object.getBytes(StandardCharsets.UTF_8);
112-
BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(bucket, key)).build();
11397
Storage.BlobWriteOption precondition = Storage.BlobWriteOption.doesNotExist();
114-
try (WriteChannel writer = storage.writer(blobInfo, precondition)) {
115-
if (parallelUploadBlockSizeInBytes != null) {
116-
writer.setChunkSize(parallelUploadBlockSizeInBytes);
117-
}
118-
ByteBuffer buffer = ByteBuffer.wrap(data);
119-
while (buffer.hasRemaining()) {
120-
writer.write(buffer);
121-
}
122-
}
98+
writeData(key, object, precondition);
12399
} catch (StorageException e) {
124100
if (e.getCode() == CloudStorageErrorCode.PRECONDITION_FAILED.get()) {
125101
throw new PreconditionFailedException(
@@ -139,19 +115,9 @@ public void insert(String key, String object) throws ObjectStorageWrapperExcepti
139115
public void update(String key, String object, String version)
140116
throws ObjectStorageWrapperException {
141117
try {
142-
byte[] data = object.getBytes(StandardCharsets.UTF_8);
143-
BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(bucket, key)).build();
144118
Storage.BlobWriteOption precondition =
145119
Storage.BlobWriteOption.generationMatch(Long.parseLong(version));
146-
try (WriteChannel writer = storage.writer(blobInfo, precondition)) {
147-
if (parallelUploadBlockSizeInBytes != null) {
148-
writer.setChunkSize(parallelUploadBlockSizeInBytes);
149-
}
150-
ByteBuffer buffer = ByteBuffer.wrap(data);
151-
while (buffer.hasRemaining()) {
152-
writer.write(buffer);
153-
}
154-
}
120+
writeData(key, object, precondition);
155121
} catch (StorageException e) {
156122
if (e.getCode() == CloudStorageErrorCode.PRECONDITION_FAILED.get()) {
157123
throw new PreconditionFailedException(
@@ -246,4 +212,20 @@ public void close() throws ObjectStorageWrapperException {
246212
throw new ObjectStorageWrapperException("Failed to close the storage wrapper", e);
247213
}
248214
}
215+
216+
private void writeData(String key, String object, Storage.BlobWriteOption precondition)
217+
throws IOException {
218+
byte[] data = object.getBytes(StandardCharsets.UTF_8);
219+
BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(bucket, key)).build();
220+
221+
try (WriteChannel writer = storage.writer(blobInfo, precondition)) {
222+
if (parallelUploadBlockSizeInBytes != null) {
223+
writer.setChunkSize(parallelUploadBlockSizeInBytes);
224+
}
225+
ByteBuffer buffer = ByteBuffer.wrap(data);
226+
while (buffer.hasRemaining()) {
227+
writer.write(buffer);
228+
}
229+
}
230+
}
249231
}

0 commit comments

Comments
 (0)