Skip to content

Commit 5fec84d

Browse files
KodaiDKodaiD
committed
Merge branch 'master' into fix-nosql-admin-repair
2 parents 948822d + f3969d9 commit 5fec84d

File tree

132 files changed

+7877
-3356
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

132 files changed

+7877
-3356
lines changed

‎.github/workflows/data-loader-ci.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: CI for Data Loader CLI
2+
3+
on:
4+
pull_request:
5+
workflow_dispatch:
6+
7+
jobs:
8+
data_loader_ci:
9+
name: Build Data Loader CLI
10+
runs-on: ubuntu-latest
11+
12+
steps:
13+
- uses: actions/checkout@v4
14+
15+
- name: Set up JDK 8
16+
uses: actions/setup-java@v4
17+
with:
18+
distribution: 'temurin'
19+
java-version: 8
20+
21+
- name: Setup Gradle
22+
uses: gradle/actions/setup-gradle@v4
23+
24+
- name: Gradle Data Loader CLI build
25+
run: ./gradlew :data-loader:cli:build --no-daemon --stacktrace
26+

‎build.gradle‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,12 @@ subprojects {
3030
awssdkVersion = '2.31.3'
3131
commonsDbcp2Version = '2.13.0'
3232
mysqlDriverVersion = '8.4.0'
33-
postgresqlDriverVersion = '42.7.6'
33+
postgresqlDriverVersion = '42.7.7'
3434
oracleDriverVersion = '23.8.0.25.04'
3535
sqlserverDriverVersion = '12.8.1.jre8'
3636
sqliteDriverVersion = '3.50.1.0'
3737
yugabyteDriverVersion = '42.7.3-yb-4'
38-
db2DriverVersion= '12.1.0.0'
38+
db2DriverVersion= '12.1.2.0'
3939
mariadDbDriverVersion = '3.5.3'
4040
picocliVersion = '4.7.7'
4141
commonsTextVersion = '1.13.1'

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcDatabaseIntegrationTest.java‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,23 @@
11
package com.scalar.db.storage.jdbc;
22

3+
import static org.assertj.core.api.Assertions.assertThat;
4+
5+
import com.scalar.db.api.DistributedStorage;
36
import com.scalar.db.api.DistributedStorageIntegrationTestBase;
7+
import com.scalar.db.api.Get;
8+
import com.scalar.db.api.Put;
9+
import com.scalar.db.api.Result;
10+
import com.scalar.db.api.Scan;
11+
import com.scalar.db.api.Scanner;
412
import com.scalar.db.config.DatabaseConfig;
13+
import com.scalar.db.exception.storage.ExecutionException;
14+
import com.scalar.db.io.Key;
15+
import com.scalar.db.service.StorageFactory;
16+
import java.io.IOException;
17+
import java.util.List;
18+
import java.util.Optional;
519
import java.util.Properties;
20+
import org.junit.jupiter.api.Test;
621

722
public class JdbcDatabaseIntegrationTest extends DistributedStorageIntegrationTestBase {
823

@@ -25,4 +40,113 @@ protected int getLargeDataSizeInBytes() {
2540
return super.getLargeDataSizeInBytes();
2641
}
2742
}
43+
44+
@Test
45+
public void get_InStreamingMode_ShouldRetrieveSingleResult() throws ExecutionException {
46+
if (!JdbcTestUtils.isMysql(rdbEngine) || JdbcTestUtils.isMariaDB(rdbEngine)) {
47+
// MySQL is the only RDB engine that supports streaming mode
48+
return;
49+
}
50+
51+
try (DistributedStorage storage = getStorageInStreamingMode()) {
52+
// Arrange
53+
int pKey = 0;
54+
int cKey = 1;
55+
int value = 2;
56+
57+
storage.put(
58+
Put.newBuilder()
59+
.namespace(namespace)
60+
.table(TABLE)
61+
.partitionKey(Key.ofInt(COL_NAME1, pKey))
62+
.clusteringKey(Key.ofInt(COL_NAME4, cKey))
63+
.intValue(COL_NAME3, value)
64+
.build());
65+
66+
// Act
67+
Optional<Result> result =
68+
storage.get(
69+
Get.newBuilder()
70+
.namespace(namespace)
71+
.table(TABLE)
72+
.partitionKey(Key.ofInt(COL_NAME1, pKey))
73+
.clusteringKey(Key.ofInt(COL_NAME4, cKey))
74+
.build());
75+
76+
// Assert
77+
assertThat(result.isPresent()).isTrue();
78+
assertThat(result.get().getInt(COL_NAME1)).isEqualTo(pKey);
79+
assertThat(result.get().getInt(COL_NAME4)).isEqualTo(cKey);
80+
assertThat(result.get().getInt(COL_NAME3)).isEqualTo(value);
81+
}
82+
}
83+
84+
@Test
85+
public void scan_InStreamingMode_ShouldRetrieveResults() throws IOException, ExecutionException {
86+
if (!JdbcTestUtils.isMysql(rdbEngine) || JdbcTestUtils.isMariaDB(rdbEngine)) {
87+
// MySQL is the only RDB engine that supports streaming mode
88+
return;
89+
}
90+
91+
try (DistributedStorage storage = getStorageInStreamingMode()) {
92+
// Arrange
93+
int pKey = 0;
94+
95+
storage.put(
96+
Put.newBuilder()
97+
.namespace(namespace)
98+
.table(TABLE)
99+
.partitionKey(Key.ofInt(COL_NAME1, pKey))
100+
.clusteringKey(Key.ofInt(COL_NAME4, 0))
101+
.intValue(COL_NAME3, 1)
102+
.build());
103+
storage.put(
104+
Put.newBuilder()
105+
.namespace(namespace)
106+
.table(TABLE)
107+
.partitionKey(Key.ofInt(COL_NAME1, pKey))
108+
.clusteringKey(Key.ofInt(COL_NAME4, 1))
109+
.intValue(COL_NAME3, 2)
110+
.build());
111+
storage.put(
112+
Put.newBuilder()
113+
.namespace(namespace)
114+
.table(TABLE)
115+
.partitionKey(Key.ofInt(COL_NAME1, pKey))
116+
.clusteringKey(Key.ofInt(COL_NAME4, 2))
117+
.intValue(COL_NAME3, 3)
118+
.build());
119+
120+
// Act
121+
Scanner scanner =
122+
storage.scan(
123+
Scan.newBuilder()
124+
.namespace(namespace)
125+
.table(TABLE)
126+
.partitionKey(Key.ofInt(COL_NAME1, pKey))
127+
.build());
128+
List<Result> results = scanner.all();
129+
scanner.close();
130+
131+
// Assert
132+
assertThat(results).hasSize(3);
133+
assertThat(results.get(0).getInt(COL_NAME1)).isEqualTo(pKey);
134+
assertThat(results.get(0).getInt(COL_NAME4)).isEqualTo(0);
135+
assertThat(results.get(0).getInt(COL_NAME3)).isEqualTo(1);
136+
137+
assertThat(results.get(1).getInt(COL_NAME1)).isEqualTo(pKey);
138+
assertThat(results.get(1).getInt(COL_NAME4)).isEqualTo(1);
139+
assertThat(results.get(1).getInt(COL_NAME3)).isEqualTo(2);
140+
141+
assertThat(results.get(2).getInt(COL_NAME1)).isEqualTo(pKey);
142+
assertThat(results.get(2).getInt(COL_NAME4)).isEqualTo(2);
143+
assertThat(results.get(2).getInt(COL_NAME3)).isEqualTo(3);
144+
}
145+
}
146+
147+
private DistributedStorage getStorageInStreamingMode() {
148+
Properties properties = JdbcEnv.getProperties(TEST_NAME);
149+
properties.setProperty(DatabaseConfig.SCAN_FETCH_SIZE, Integer.toString(Integer.MIN_VALUE));
150+
return StorageFactory.create(properties).getStorage();
151+
}
28152
}

‎core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcTestUtils.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,10 @@ public static boolean isDb2(RdbEngineStrategy rdbEngine) {
8686
return rdbEngine instanceof RdbEngineDb2;
8787
}
8888

89+
public static boolean isMariaDB(RdbEngineStrategy rdbEngine) {
90+
return rdbEngine instanceof RdbEngineMariaDB;
91+
}
92+
8993
/**
9094
* Filters the data types based on the RDB engine and the excluded data types.
9195
*

‎core/src/main/java/com/scalar/db/api/ConditionBuilder.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,23 @@ public static ConditionalExpression buildConditionalExpression(
135135
return new ConditionalExpression(column, operator);
136136
}
137137

138+
/**
139+
* Builds a like expression with the specified column, operator, and escape character.
140+
*
141+
* <p>This method is primarily for internal use. Breaking changes can and will be introduced to
142+
* this method. Users should not depend on it.
143+
*
144+
* @param column a target text column used to compare
145+
* @param operator an operator used to compare the target column. The operator must be either LIKE
146+
* or NOT_LIKE.
147+
* @param escape an escape character for the like operator
148+
* @return a conditional expression
149+
*/
150+
public static ConditionalExpression buildLikeExpression(
151+
TextColumn column, Operator operator, String escape) {
152+
return new LikeExpression(column, operator, escape);
153+
}
154+
138155
/**
139156
* Returns a builder object for a condition expression for PutIf/DeleteIf
140157
*
@@ -352,6 +369,7 @@ public ConditionalExpression isNotEqualToBlob(byte[] value) {
352369
public ConditionalExpression isNotEqualToBlob(ByteBuffer value) {
353370
return new ConditionalExpression(columnName, value, Operator.NE);
354371
}
372+
355373
/**
356374
* Creates a 'not equal' conditional expression for a DATE value.
357375
*
@@ -391,6 +409,7 @@ public ConditionalExpression isNotEqualToTimestamp(LocalDateTime value) {
391409
public ConditionalExpression isNotEqualToTimestampTZ(Instant value) {
392410
return new ConditionalExpression(TimestampTZColumn.of(columnName, value), Operator.NE);
393411
}
412+
394413
/**
395414
* Creates a 'greater than' conditional expression for a BOOLEAN value.
396415
*
@@ -590,6 +609,7 @@ public ConditionalExpression isGreaterThanOrEqualToBlob(byte[] value) {
590609
public ConditionalExpression isGreaterThanOrEqualToBlob(ByteBuffer value) {
591610
return new ConditionalExpression(columnName, value, Operator.GTE);
592611
}
612+
593613
/**
594614
* Creates a 'greater than or equal' conditional expression for a DATE value.
595615
*
@@ -709,6 +729,7 @@ public ConditionalExpression isLessThanBlob(byte[] value) {
709729
public ConditionalExpression isLessThanBlob(ByteBuffer value) {
710730
return new ConditionalExpression(columnName, value, Operator.LT);
711731
}
732+
712733
/**
713734
* Creates a 'less than' conditional expression for a DATE value.
714735
*
@@ -748,6 +769,7 @@ public ConditionalExpression isLessThanTimestamp(LocalDateTime value) {
748769
public ConditionalExpression isLessThanTimestampTZ(Instant value) {
749770
return new ConditionalExpression(TimestampTZColumn.of(columnName, value), Operator.LT);
750771
}
772+
751773
/**
752774
* Creates a 'less than or equal' conditional expression for a BOOLEAN value.
753775
*
@@ -1029,6 +1051,7 @@ public ConditionalExpression isNotNullText() {
10291051
public ConditionalExpression isNotNullBlob() {
10301052
return new ConditionalExpression(BlobColumn.ofNull(columnName), Operator.IS_NOT_NULL);
10311053
}
1054+
10321055
/**
10331056
* Creates an 'is not null' conditional expression for a DATE value.
10341057
*

‎core/src/main/java/com/scalar/db/api/LikeExpression.java‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,9 @@ public class LikeExpression extends ConditionalExpression {
1616
* Constructs a {@code LikeExpression} with the specified column and operator. For the escape
1717
* character, the default one ("\", i.e., backslash) is used.
1818
*
19-
* @param column a target column used to compare
20-
* @param operator an operator used to compare the target column
19+
* @param column a target text column used to compare
20+
* @param operator an operator used to compare the target text column. The operator must be either
21+
* LIKE or NOT_LIKE.
2122
*/
2223
LikeExpression(TextColumn column, Operator operator) {
2324
this(column, operator, DEFAULT_ESCAPE_CHAR);
@@ -28,8 +29,9 @@ public class LikeExpression extends ConditionalExpression {
2829
* The escape character must be a string of a single character or an empty string. If an empty
2930
* string is specified, the escape character is disabled.
3031
*
31-
* @param column a target column used to compare
32-
* @param operator an operator used to compare the target column
32+
* @param column a target text column used to compare
33+
* @param operator an operator used to compare the target text column. The operator must be either
34+
* LIKE or NOT_LIKE.
3335
* @param escape an escape character for the like operator
3436
*/
3537
LikeExpression(TextColumn column, Operator operator, String escape) {
@@ -75,6 +77,11 @@ private void check(String pattern, Operator operator, String escape) {
7577
}
7678
}
7779

80+
@Override
81+
public TextColumn getColumn() {
82+
return (TextColumn) super.getColumn();
83+
}
84+
7885
/**
7986
* Returns the escape character for LIKE operator.
8087
*

‎core/src/main/java/com/scalar/db/common/error/CoreError.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1241,6 +1241,8 @@ public enum CoreError implements ScalarDbError {
12411241
"Getting the storage information failed. Namespace: %s",
12421242
"",
12431243
""),
1244+
CONSENSUS_COMMIT_RECOVERING_RECORDS_FAILED(
1245+
Category.INTERNAL_ERROR, "0057", "Recovering records failed. Details: %s", "", ""),
12441246

12451247
//
12461248
// Errors for the unknown transaction status error category

‎core/src/main/java/com/scalar/db/storage/dynamo/DynamoAdmin.java‎

Lines changed: 43 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ public class DynamoAdmin implements DistributedStorageAdmin {
9797
public static final String DEFAULT_NO_BACKUP = "false";
9898
public static final String DEFAULT_REQUEST_UNIT = "10";
9999
private static final int DEFAULT_WAITING_DURATION_SECS = 3;
100+
@VisibleForTesting static final int MAX_RETRY_COUNT = 10;
100101

101102
@VisibleForTesting static final String PARTITION_KEY = "concatenatedPartitionKey";
102103
@VisibleForTesting static final String CLUSTERING_KEY = "concatenatedClusteringKey";
@@ -238,15 +239,26 @@ public void createNamespace(String nonPrefixedNamespace, Map<String, String> opt
238239
private void upsertIntoNamespacesTable(Namespace namespace) throws ExecutionException {
239240
Map<String, AttributeValue> itemValues = new HashMap<>();
240241
itemValues.put(NAMESPACES_ATTR_NAME, AttributeValue.builder().s(namespace.prefixed()).build());
241-
try {
242-
client.putItem(
243-
PutItemRequest.builder()
244-
.tableName(ScalarDbUtils.getFullTableName(metadataNamespace, NAMESPACES_TABLE))
245-
.item(itemValues)
246-
.build());
247-
} catch (Exception e) {
248-
throw new ExecutionException(
249-
"Inserting the " + namespace + " namespace into the namespaces table failed", e);
242+
int retryCount = 0;
243+
while (true) {
244+
try {
245+
client.putItem(
246+
PutItemRequest.builder()
247+
.tableName(ScalarDbUtils.getFullTableName(metadataNamespace, NAMESPACES_TABLE))
248+
.item(itemValues)
249+
.build());
250+
return;
251+
} catch (ResourceNotFoundException e) {
252+
if (retryCount >= MAX_RETRY_COUNT) {
253+
throw new ExecutionException(
254+
"Inserting the " + namespace + " namespace into the namespaces table failed", e);
255+
}
256+
Uninterruptibles.sleepUninterruptibly(waitingDurationSecs, TimeUnit.SECONDS);
257+
retryCount++;
258+
} catch (Exception e) {
259+
throw new ExecutionException(
260+
"Inserting the " + namespace + " namespace into the namespaces table failed", e);
261+
}
250262
}
251263
}
252264

@@ -470,15 +482,28 @@ private void upsertTableMetadata(Namespace namespace, String table, TableMetadat
470482
METADATA_ATTR_SECONDARY_INDEX,
471483
AttributeValue.builder().ss(metadata.getSecondaryIndexNames()).build());
472484
}
473-
try {
474-
client.putItem(
475-
PutItemRequest.builder()
476-
.tableName(ScalarDbUtils.getFullTableName(metadataNamespace, METADATA_TABLE))
477-
.item(itemValues)
478-
.build());
479-
} catch (Exception e) {
480-
throw new ExecutionException(
481-
"Adding the metadata for the " + getFullTableName(namespace, table) + " table failed", e);
485+
int retryCount = 0;
486+
while (true) {
487+
try {
488+
client.putItem(
489+
PutItemRequest.builder()
490+
.tableName(ScalarDbUtils.getFullTableName(metadataNamespace, METADATA_TABLE))
491+
.item(itemValues)
492+
.build());
493+
return;
494+
} catch (ResourceNotFoundException e) {
495+
if (retryCount >= MAX_RETRY_COUNT) {
496+
throw new ExecutionException(
497+
"Adding the metadata for the " + getFullTableName(namespace, table) + " table failed",
498+
e);
499+
}
500+
Uninterruptibles.sleepUninterruptibly(waitingDurationSecs, TimeUnit.SECONDS);
501+
retryCount++;
502+
} catch (Exception e) {
503+
throw new ExecutionException(
504+
"Adding the metadata for the " + getFullTableName(namespace, table) + " table failed",
505+
e);
506+
}
482507
}
483508
}
484509

0 commit comments

Comments
 (0)