Skip to content

Commit 2361f43

Browse files
feeblefakieinv-jishnuypeckstadt
authored
Backport to branch(3) : Add export options validator (#2453)
Co-authored-by: inv-jishnu <[email protected]> Co-authored-by: Peckstadt Yves <[email protected]>
1 parent a223a08 commit 2361f43

File tree

4 files changed

+375
-0
lines changed

4 files changed

+375
-0
lines changed

‎core/src/main/java/com/scalar/db/common/error/CoreError.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -696,6 +696,28 @@ public enum CoreError implements ScalarDbError {
696696
"The attribute-based access control feature is not enabled. To use this feature, you must enable it. Note that this feature is supported only in the ScalarDB Enterprise edition",
697697
"",
698698
""),
699+
DATA_LOADER_CLUSTERING_KEY_NOT_FOUND(
700+
Category.USER_ERROR, "0153", "The provided clustering key %s was not found", "", ""),
701+
DATA_LOADER_INVALID_PROJECTION(
702+
Category.USER_ERROR, "0154", "The column '%s' was not found", "", ""),
703+
DATA_LOADER_INCOMPLETE_PARTITION_KEY(
704+
Category.USER_ERROR,
705+
"0155",
706+
"The provided partition key is incomplete. Required key: %s",
707+
"",
708+
""),
709+
DATA_LOADER_CLUSTERING_KEY_ORDER_MISMATCH(
710+
Category.USER_ERROR,
711+
"0156",
712+
"The provided clustering key order does not match the table schema. Required order: %s",
713+
"",
714+
""),
715+
DATA_LOADER_PARTITION_KEY_ORDER_MISMATCH(
716+
Category.USER_ERROR,
717+
"0157",
718+
"The provided partition key order does not match the table schema. Required order: %s",
719+
"",
720+
""),
699721

700722
//
701723
// Errors for the concurrency error category

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/validation/ExportOptionsValidationException.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package com.scalar.db.dataloader.core.dataexport.validation;
2+
3+
/** A custom exception for export options validation errors */
4+
public class ExportOptionsValidationException extends Exception {
5+
6+
/**
7+
* Class constructor
8+
*
9+
* @param message error message
10+
*/
11+
public ExportOptionsValidationException(String message) {
12+
super(message);
13+
}
14+
}

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/validation/ExportOptionsValidator.java‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
package com.scalar.db.dataloader.core.dataexport.validation;
2+
3+
import com.scalar.db.api.Scan;
4+
import com.scalar.db.api.TableMetadata;
5+
import com.scalar.db.common.error.CoreError;
6+
import com.scalar.db.dataloader.core.ScanRange;
7+
import com.scalar.db.dataloader.core.dataexport.ExportOptions;
8+
import com.scalar.db.io.Column;
9+
import com.scalar.db.io.Key;
10+
import java.util.Iterator;
11+
import java.util.LinkedHashSet;
12+
import java.util.List;
13+
import lombok.AccessLevel;
14+
import lombok.NoArgsConstructor;
15+
16+
/**
17+
* A validator for ensuring that export options are consistent with the ScalarDB table metadata and
18+
* follow the defined constraints.
19+
*/
20+
@NoArgsConstructor(access = AccessLevel.PRIVATE)
21+
public class ExportOptionsValidator {
22+
23+
/**
24+
* Validates the export request.
25+
*
26+
* @param exportOptions The export options provided by the user.
27+
* @param tableMetadata The metadata of the ScalarDB table to validate against.
28+
* @throws ExportOptionsValidationException If the export options are invalid.
29+
*/
30+
public static void validate(ExportOptions exportOptions, TableMetadata tableMetadata)
31+
throws ExportOptionsValidationException {
32+
LinkedHashSet<String> partitionKeyNames = tableMetadata.getPartitionKeyNames();
33+
LinkedHashSet<String> clusteringKeyNames = tableMetadata.getClusteringKeyNames();
34+
ScanRange scanRange = exportOptions.getScanRange();
35+
36+
validatePartitionKey(partitionKeyNames, exportOptions.getScanPartitionKey());
37+
validateProjectionColumns(tableMetadata.getColumnNames(), exportOptions.getProjectionColumns());
38+
validateSortOrders(clusteringKeyNames, exportOptions.getSortOrders());
39+
40+
if (scanRange.getScanStartKey() != null) {
41+
validateClusteringKey(clusteringKeyNames, scanRange.getScanStartKey());
42+
}
43+
if (scanRange.getScanEndKey() != null) {
44+
validateClusteringKey(clusteringKeyNames, scanRange.getScanEndKey());
45+
}
46+
}
47+
48+
/*
49+
* Check if the provided partition key is available in the ScalarDB table
50+
* @param partitionKeyNames List of partition key names available in a
51+
* @param key To be validated ScalarDB key
52+
* @throws ExportOptionsValidationException if the key could not be found or is not a partition
53+
*/
54+
private static void validatePartitionKey(LinkedHashSet<String> partitionKeyNames, Key key)
55+
throws ExportOptionsValidationException {
56+
if (partitionKeyNames == null || key == null) {
57+
return;
58+
}
59+
60+
// Make sure that all partition key columns are provided
61+
if (partitionKeyNames.size() != key.getColumns().size()) {
62+
throw new ExportOptionsValidationException(
63+
CoreError.DATA_LOADER_INCOMPLETE_PARTITION_KEY.buildMessage(partitionKeyNames));
64+
}
65+
66+
// Check if the order of columns in key.getColumns() matches the order in partitionKeyNames
67+
Iterator<String> partitionKeyIterator = partitionKeyNames.iterator();
68+
for (Column<?> column : key.getColumns()) {
69+
// Check if the column names match in order
70+
if (!partitionKeyIterator.hasNext()
71+
|| !partitionKeyIterator.next().equals(column.getName())) {
72+
throw new ExportOptionsValidationException(
73+
CoreError.DATA_LOADER_PARTITION_KEY_ORDER_MISMATCH.buildMessage(partitionKeyNames));
74+
}
75+
}
76+
}
77+
78+
private static void validateSortOrders(
79+
LinkedHashSet<String> clusteringKeyNames, List<Scan.Ordering> sortOrders)
80+
throws ExportOptionsValidationException {
81+
if (sortOrders == null || sortOrders.isEmpty()) {
82+
return;
83+
}
84+
85+
for (Scan.Ordering sortOrder : sortOrders) {
86+
checkIfColumnExistsAsClusteringKey(clusteringKeyNames, sortOrder.getColumnName());
87+
}
88+
}
89+
90+
/**
91+
* Validates that the clustering key columns in the given Key object match the expected order
92+
* defined in the clusteringKeyNames. The Key can be a prefix of the clusteringKeyNames, but the
93+
* order must remain consistent.
94+
*
95+
* @param clusteringKeyNames the expected ordered set of clustering key names
96+
* @param key the Key object containing the actual clustering key columns
97+
* @throws ExportOptionsValidationException if the order or names of clustering keys do not match
98+
*/
99+
private static void validateClusteringKey(LinkedHashSet<String> clusteringKeyNames, Key key)
100+
throws ExportOptionsValidationException {
101+
// If either clusteringKeyNames or key is null, no validation is needed
102+
if (clusteringKeyNames == null || key == null) {
103+
return;
104+
}
105+
106+
// Create an iterator to traverse the clusteringKeyNames in order
107+
Iterator<String> clusteringKeyIterator = clusteringKeyNames.iterator();
108+
109+
// Iterate through the columns in the given Key
110+
for (Column<?> column : key.getColumns()) {
111+
// If clusteringKeyNames have been exhausted but columns still exist in the Key,
112+
// it indicates a mismatch
113+
if (!clusteringKeyIterator.hasNext()) {
114+
throw new ExportOptionsValidationException(
115+
CoreError.DATA_LOADER_CLUSTERING_KEY_ORDER_MISMATCH.buildMessage(clusteringKeyNames));
116+
}
117+
118+
// Get the next expected clustering key name
119+
String expectedKey = clusteringKeyIterator.next();
120+
121+
// Check if the current column name matches the expected clustering key name
122+
if (!column.getName().equals(expectedKey)) {
123+
throw new ExportOptionsValidationException(
124+
CoreError.DATA_LOADER_CLUSTERING_KEY_ORDER_MISMATCH.buildMessage(clusteringKeyNames));
125+
}
126+
}
127+
}
128+
129+
private static void checkIfColumnExistsAsClusteringKey(
130+
LinkedHashSet<String> clusteringKeyNames, String columnName)
131+
throws ExportOptionsValidationException {
132+
if (clusteringKeyNames == null || columnName == null) {
133+
return;
134+
}
135+
136+
if (!clusteringKeyNames.contains(columnName)) {
137+
throw new ExportOptionsValidationException(
138+
CoreError.DATA_LOADER_CLUSTERING_KEY_NOT_FOUND.buildMessage(columnName));
139+
}
140+
}
141+
142+
private static void validateProjectionColumns(
143+
LinkedHashSet<String> columnNames, List<String> columns)
144+
throws ExportOptionsValidationException {
145+
if (columns == null || columns.isEmpty()) {
146+
return;
147+
}
148+
149+
for (String column : columns) {
150+
if (!columnNames.contains(column)) {
151+
throw new ExportOptionsValidationException(
152+
CoreError.DATA_LOADER_INVALID_PROJECTION.buildMessage(column));
153+
}
154+
}
155+
}
156+
}

‎data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/validation/ExportOptionsValidatorTest.java‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
package com.scalar.db.dataloader.core.dataexport.validation;
2+
3+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
4+
5+
import com.scalar.db.api.TableMetadata;
6+
import com.scalar.db.common.error.CoreError;
7+
import com.scalar.db.dataloader.core.FileFormat;
8+
import com.scalar.db.dataloader.core.ScanRange;
9+
import com.scalar.db.dataloader.core.dataexport.ExportOptions;
10+
import com.scalar.db.io.DataType;
11+
import com.scalar.db.io.IntColumn;
12+
import com.scalar.db.io.Key;
13+
import com.scalar.db.io.TextColumn;
14+
import java.util.ArrayList;
15+
import java.util.Collections;
16+
import java.util.LinkedHashSet;
17+
import java.util.List;
18+
import org.junit.jupiter.api.BeforeEach;
19+
import org.junit.jupiter.api.Test;
20+
21+
class ExportOptionsValidatorTest {
22+
23+
private TableMetadata singlePkCkMetadata;
24+
private TableMetadata multiplePkCkMetadata;
25+
private List<String> projectedColumns;
26+
27+
@BeforeEach
28+
void setup() {
29+
singlePkCkMetadata = createMockMetadata(1, 1);
30+
multiplePkCkMetadata = createMockMetadata(2, 2);
31+
projectedColumns = createProjectedColumns();
32+
}
33+
34+
private TableMetadata createMockMetadata(int pkCount, int ckCount) {
35+
TableMetadata.Builder builder = TableMetadata.newBuilder();
36+
37+
// Add partition keys
38+
for (int i = 1; i <= pkCount; i++) {
39+
builder.addColumn("pk" + i, DataType.INT);
40+
builder.addPartitionKey("pk" + i);
41+
}
42+
43+
// Add clustering keys
44+
for (int i = 1; i <= ckCount; i++) {
45+
builder.addColumn("ck" + i, DataType.TEXT);
46+
builder.addClusteringKey("ck" + i);
47+
}
48+
49+
return builder.build();
50+
}
51+
52+
private List<String> createProjectedColumns() {
53+
List<String> columns = new ArrayList<>();
54+
columns.add("pk1");
55+
columns.add("ck1");
56+
return columns;
57+
}
58+
59+
@Test
60+
void validate_withValidExportOptionsForSinglePkCk_ShouldNotThrowException()
61+
throws ExportOptionsValidationException {
62+
63+
Key partitionKey = Key.newBuilder().add(IntColumn.of("pk1", 1)).build();
64+
65+
ExportOptions exportOptions =
66+
ExportOptions.builder("test", "sample", partitionKey, FileFormat.JSON)
67+
.projectionColumns(projectedColumns)
68+
.scanRange(new ScanRange(null, null, false, false))
69+
.build();
70+
71+
ExportOptionsValidator.validate(exportOptions, singlePkCkMetadata);
72+
}
73+
74+
@Test
75+
void validate_withValidExportOptionsForMultiplePkCk_ShouldNotThrowException()
76+
throws ExportOptionsValidationException {
77+
78+
Key partitionKey =
79+
Key.newBuilder().add(IntColumn.of("pk1", 1)).add(IntColumn.of("pk2", 2)).build();
80+
81+
ExportOptions exportOptions =
82+
ExportOptions.builder("test", "sample", partitionKey, FileFormat.JSON)
83+
.projectionColumns(projectedColumns)
84+
.scanRange(new ScanRange(null, null, false, false))
85+
.build();
86+
87+
ExportOptionsValidator.validate(exportOptions, multiplePkCkMetadata);
88+
}
89+
90+
@Test
91+
void validate_withIncompletePartitionKeyForSinglePk_ShouldThrowException() {
92+
Key incompletePartitionKey = Key.newBuilder().build();
93+
94+
ExportOptions exportOptions =
95+
ExportOptions.builder("test", "sample", incompletePartitionKey, FileFormat.JSON).build();
96+
97+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, singlePkCkMetadata))
98+
.isInstanceOf(ExportOptionsValidationException.class)
99+
.hasMessage(
100+
CoreError.DATA_LOADER_INCOMPLETE_PARTITION_KEY.buildMessage(
101+
singlePkCkMetadata.getPartitionKeyNames()));
102+
}
103+
104+
@Test
105+
void validate_withIncompletePartitionKeyForMultiplePks_ShouldThrowException() {
106+
Key incompletePartitionKey = Key.newBuilder().add(IntColumn.of("pk1", 1)).build();
107+
108+
ExportOptions exportOptions =
109+
ExportOptions.builder("test", "sample", incompletePartitionKey, FileFormat.JSON).build();
110+
111+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, multiplePkCkMetadata))
112+
.isInstanceOf(ExportOptionsValidationException.class)
113+
.hasMessage(
114+
CoreError.DATA_LOADER_INCOMPLETE_PARTITION_KEY.buildMessage(
115+
multiplePkCkMetadata.getPartitionKeyNames()));
116+
}
117+
118+
@Test
119+
void validate_withInvalidProjectionColumn_ShouldThrowException() {
120+
ExportOptions exportOptions =
121+
ExportOptions.builder(
122+
"test",
123+
"sample",
124+
Key.newBuilder().add(IntColumn.of("pk1", 1)).build(),
125+
FileFormat.JSON)
126+
.projectionColumns(Collections.singletonList("invalid_column"))
127+
.build();
128+
129+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, singlePkCkMetadata))
130+
.isInstanceOf(ExportOptionsValidationException.class)
131+
.hasMessage(CoreError.DATA_LOADER_INVALID_PROJECTION.buildMessage("invalid_column"));
132+
}
133+
134+
@Test
135+
void validate_withInvalidClusteringKeyInScanRange_ShouldThrowException() {
136+
ScanRange scanRange =
137+
new ScanRange(
138+
Key.newBuilder().add(TextColumn.of("invalid_ck", "value")).build(),
139+
Key.newBuilder().add(TextColumn.of("ck1", "value")).build(),
140+
false,
141+
false);
142+
143+
ExportOptions exportOptions =
144+
ExportOptions.builder("test", "sample", createValidPartitionKey(), FileFormat.JSON)
145+
.scanRange(scanRange)
146+
.build();
147+
148+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, singlePkCkMetadata))
149+
.isInstanceOf(ExportOptionsValidationException.class)
150+
.hasMessage(CoreError.DATA_LOADER_CLUSTERING_KEY_ORDER_MISMATCH.buildMessage("[ck1]"));
151+
}
152+
153+
@Test
154+
void validate_withInvalidPartitionKeyOrder_ShouldThrowException() {
155+
// Partition key names are expected to be "pk1", "pk2"
156+
LinkedHashSet<String> partitionKeyNames = new LinkedHashSet<>();
157+
partitionKeyNames.add("pk1");
158+
partitionKeyNames.add("pk2");
159+
160+
// Create a partition key with reversed order, expecting an error
161+
Key invalidPartitionKey =
162+
Key.newBuilder()
163+
.add(IntColumn.of("pk2", 2)) // Incorrect order
164+
.add(IntColumn.of("pk1", 1)) // Incorrect order
165+
.build();
166+
167+
ExportOptions exportOptions =
168+
ExportOptions.builder("test", "sample", invalidPartitionKey, FileFormat.JSON)
169+
.projectionColumns(projectedColumns)
170+
.scanRange(new ScanRange(null, null, false, false))
171+
.build();
172+
173+
// Verify that the validator throws the correct exception
174+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, multiplePkCkMetadata))
175+
.isInstanceOf(ExportOptionsValidationException.class)
176+
.hasMessage(
177+
CoreError.DATA_LOADER_PARTITION_KEY_ORDER_MISMATCH.buildMessage(partitionKeyNames));
178+
}
179+
180+
private Key createValidPartitionKey() {
181+
return Key.newBuilder().add(IntColumn.of("pk1", 1)).build();
182+
}
183+
}

0 commit comments

Comments
 (0)