Skip to content

Commit f1858f6

Browse files
KodaiDKodaiD
authored
Fix to increase the maximum allowed string length with Object Storage (#3248)
1 parent efa0b69 commit f1858f6

File tree

5 files changed

+122
-2
lines changed

5 files changed

+122
-2
lines changed

‎core/src/main/java/com/scalar/db/common/CoreError.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1028,6 +1028,12 @@ public enum CoreError implements ScalarDbError {
10281028
+ "you may be able to adjust the settings to enable consistent reads. Please refer to the storage configuration for details. Storage: %s",
10291029
"",
10301030
""),
1031+
OBJECT_STORAGE_BLOB_EXCEEDS_MAX_LENGTH_ALLOWED(
1032+
Category.USER_ERROR,
1033+
"0279",
1034+
"The size of a BLOB column value exceeds the maximum allowed size of %d bytes. Column: %s; Size: %d bytes",
1035+
"",
1036+
""),
10311037

10321038
//
10331039
// Errors for the concurrency error category

‎core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,61 @@
2323
import com.scalar.db.io.TimeColumn;
2424
import com.scalar.db.io.TimestampColumn;
2525
import com.scalar.db.io.TimestampTZColumn;
26+
import java.nio.ByteBuffer;
2627

2728
public class ObjectStorageOperationChecker extends OperationChecker {
2829
private static final char[] ILLEGAL_CHARACTERS_IN_PRIMARY_KEY = {
2930
ObjectStorageUtils.OBJECT_KEY_DELIMITER, ObjectStorageUtils.CONCATENATED_KEY_DELIMITER,
3031
};
3132

33+
private static final ColumnVisitor COMMON_COLUMN_CHECKER =
34+
new ColumnVisitor() {
35+
@Override
36+
public void visit(BooleanColumn column) {}
37+
38+
@Override
39+
public void visit(IntColumn column) {}
40+
41+
@Override
42+
public void visit(BigIntColumn column) {}
43+
44+
@Override
45+
public void visit(FloatColumn column) {}
46+
47+
@Override
48+
public void visit(DoubleColumn column) {}
49+
50+
@Override
51+
public void visit(TextColumn column) {}
52+
53+
@Override
54+
public void visit(BlobColumn column) {
55+
ByteBuffer buffer = column.getBlobValue();
56+
if (buffer == null) {
57+
return;
58+
}
59+
// Calculate the maximum allowed blob length after Base64 encoding.
60+
long allowedLength = (long) Serializer.MAX_STRING_LENGTH_ALLOWED / 4 * 3;
61+
if (buffer.remaining() > allowedLength) {
62+
throw new IllegalArgumentException(
63+
CoreError.OBJECT_STORAGE_BLOB_EXCEEDS_MAX_LENGTH_ALLOWED.buildMessage(
64+
Serializer.MAX_STRING_LENGTH_ALLOWED, column.getName(), buffer.remaining()));
65+
}
66+
}
67+
68+
@Override
69+
public void visit(DateColumn column) {}
70+
71+
@Override
72+
public void visit(TimeColumn column) {}
73+
74+
@Override
75+
public void visit(TimestampColumn column) {}
76+
77+
@Override
78+
public void visit(TimestampTZColumn column) {}
79+
};
80+
3281
private static final ColumnVisitor PRIMARY_KEY_COLUMN_CHECKER =
3382
new ColumnVisitor() {
3483
@Override
@@ -104,6 +153,7 @@ public void check(Scan scan) throws ExecutionException {
104153
@Override
105154
public void check(Put put) throws ExecutionException {
106155
super.check(put);
156+
put.getColumns().values().forEach(column -> column.accept(COMMON_COLUMN_CHECKER));
107157
checkPrimaryKey(put);
108158
}
109159

‎core/src/main/java/com/scalar/db/storage/objectstorage/Serializer.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,21 @@
11
package com.scalar.db.storage.objectstorage;
22

3+
import com.fasterxml.jackson.core.StreamReadConstraints;
34
import com.fasterxml.jackson.core.type.TypeReference;
45
import com.fasterxml.jackson.databind.DeserializationFeature;
56
import com.fasterxml.jackson.databind.ObjectMapper;
67
import com.fasterxml.jackson.databind.SerializationFeature;
78
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
89

910
public class Serializer {
11+
public static final int MAX_STRING_LENGTH_ALLOWED = Integer.MAX_VALUE;
1012
private static final ObjectMapper mapper = new ObjectMapper();
1113

1214
static {
15+
mapper
16+
.getFactory()
17+
.setStreamReadConstraints(
18+
StreamReadConstraints.builder().maxStringLength(MAX_STRING_LENGTH_ALLOWED).build());
1319
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
1420
mapper.configure(SerializationFeature.WRAP_ROOT_VALUE, false);
1521
mapper.registerModule(new JavaTimeModule());

‎core/src/test/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationCheckerTest.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import static org.assertj.core.api.Assertions.assertThatCode;
1010
import static org.assertj.core.api.Assertions.assertThatThrownBy;
1111
import static org.mockito.ArgumentMatchers.any;
12+
import static org.mockito.Mockito.mock;
13+
import static org.mockito.Mockito.spy;
1214
import static org.mockito.Mockito.when;
1315
import static org.mockito.MockitoAnnotations.openMocks;
1416

@@ -24,9 +26,14 @@
2426
import com.scalar.db.common.TableMetadataManager;
2527
import com.scalar.db.config.DatabaseConfig;
2628
import com.scalar.db.exception.storage.ExecutionException;
29+
import com.scalar.db.io.BlobColumn;
30+
import com.scalar.db.io.Column;
2731
import com.scalar.db.io.DataType;
2832
import com.scalar.db.io.Key;
33+
import java.nio.ByteBuffer;
2934
import java.util.Arrays;
35+
import java.util.LinkedHashMap;
36+
import java.util.Map;
3037
import org.junit.jupiter.api.BeforeEach;
3138
import org.junit.jupiter.api.Test;
3239
import org.mockito.Mock;
@@ -807,6 +814,57 @@ public void check_ForMutationsWithDeleteWithCondition_ShouldBehaveProperly()
807814
.doesNotThrowAnyException();
808815
}
809816

817+
@Test
818+
public void check_PutGiven_WhenBlobColumnIsWithinLimit_ShouldNotThrowException()
819+
throws ExecutionException {
820+
// Arrange
821+
when(metadataManager.getTableMetadata(any())).thenReturn(TABLE_METADATA1);
822+
823+
byte[] blob = new byte[100];
824+
Put put =
825+
Put.newBuilder()
826+
.namespace(NAMESPACE_NAME)
827+
.table(TABLE_NAME)
828+
.partitionKey(Key.ofInt(PKEY1, 0))
829+
.clusteringKey(Key.ofInt(CKEY1, 0))
830+
.blobValue(COL4, blob)
831+
.build();
832+
833+
// Act Assert
834+
assertThatCode(() -> operationChecker.check(put)).doesNotThrowAnyException();
835+
}
836+
837+
@Test
838+
public void check_PutGiven_WhenBlobColumnExceedsLimit_ShouldThrowIllegalArgumentException()
839+
throws ExecutionException {
840+
// Arrange
841+
when(metadataManager.getTableMetadata(any())).thenReturn(TABLE_METADATA1);
842+
843+
int allowedLength = Serializer.MAX_STRING_LENGTH_ALLOWED / 4 * 3;
844+
ByteBuffer mockBuffer = mock(ByteBuffer.class);
845+
when(mockBuffer.remaining()).thenReturn(allowedLength + 1);
846+
847+
BlobColumn blobColumn = mock(BlobColumn.class);
848+
when(blobColumn.getName()).thenReturn(COL4);
849+
when(blobColumn.getBlobValue()).thenReturn(mockBuffer);
850+
851+
Put put =
852+
spy(
853+
Put.newBuilder()
854+
.namespace(NAMESPACE_NAME)
855+
.table(TABLE_NAME)
856+
.partitionKey(Key.ofInt(PKEY1, 0))
857+
.clusteringKey(Key.ofInt(CKEY1, 0))
858+
.build());
859+
Map<String, Column<?>> columns = new LinkedHashMap<>();
860+
columns.put(COL4, blobColumn);
861+
when(put.getColumns()).thenReturn(columns);
862+
863+
// Act Assert
864+
assertThatThrownBy(() -> operationChecker.check(put))
865+
.isInstanceOf(IllegalArgumentException.class);
866+
}
867+
810868
private Put buildPutWithCondition(MutationCondition condition) {
811869
return Put.newBuilder()
812870
.namespace(NAMESPACE_NAME)

‎integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2396,7 +2396,7 @@ private void populateRecords() {
23962396
puts.forEach(p -> assertThatCode(() -> storage.put(p)).doesNotThrowAnyException());
23972397
}
23982398

2399-
private Get prepareGet(int pKey, int cKey) {
2399+
protected Get prepareGet(int pKey, int cKey) {
24002400
Key partitionKey = Key.ofInt(getColumnName1(), pKey);
24012401
Key clusteringKey = Key.ofInt(getColumnName4(), cKey);
24022402
return Get.newBuilder()
@@ -2407,7 +2407,7 @@ private Get prepareGet(int pKey, int cKey) {
24072407
.build();
24082408
}
24092409

2410-
private List<Put> preparePuts() {
2410+
protected List<Put> preparePuts() {
24112411
List<Put> puts = new ArrayList<>();
24122412

24132413
IntStream.range(0, 5)

0 commit comments

Comments
 (0)