Skip to content

Commit 840c312

Browse files
feeblefakieinv-jishnuypeckstadt
authored
Backport to branch(3.11) : Add export options validator (#2457)
Co-authored-by: inv-jishnu <[email protected]> Co-authored-by: Peckstadt Yves <[email protected]>
1 parent 6a6a0b9 commit 840c312

File tree

4 files changed

+359
-0
lines changed

4 files changed

+359
-0
lines changed

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/ErrorMessage.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,13 @@ public class ErrorMessage {
88
public static final String INVALID_COLUMN_NON_EXISTENT =
99
"Invalid key: Column %s does not exist in the table %s in namespace %s.";
1010
public static final String ERROR_METHOD_NULL_ARGUMENT = "Method null argument not allowed";
11+
public static final String PARTITION_KEY_ORDER_MISMATCH =
12+
"The provided partition key order does not match the table schema. Required order: %s";
13+
public static final String INCOMPLETE_PARTITION_KEY =
14+
"The provided partition key is incomplete. Required key: %s";
15+
public static final String CLUSTERING_KEY_ORDER_MISMATCH =
16+
"The provided clustering key order does not match the table schema. Required order: %s";
17+
public static final String CLUSTERING_KEY_NOT_FOUND =
18+
"The provided clustering key %s was not found";
19+
public static final String INVALID_PROJECTION = "The column '%s' was not found";
1120
}

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/validation/ExportOptionsValidationException.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package com.scalar.db.dataloader.core.dataexport.validation;
2+
3+
/** A custom exception for export options validation errors */
4+
public class ExportOptionsValidationException extends Exception {
5+
6+
/**
7+
* Class constructor
8+
*
9+
* @param message error message
10+
*/
11+
public ExportOptionsValidationException(String message) {
12+
super(message);
13+
}
14+
}

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/validation/ExportOptionsValidator.java‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
package com.scalar.db.dataloader.core.dataexport.validation;
2+
3+
import static com.scalar.db.dataloader.core.ErrorMessage.*;
4+
5+
import com.scalar.db.api.Scan;
6+
import com.scalar.db.api.TableMetadata;
7+
import com.scalar.db.dataloader.core.ScanRange;
8+
import com.scalar.db.dataloader.core.dataexport.ExportOptions;
9+
import com.scalar.db.io.Column;
10+
import com.scalar.db.io.Key;
11+
import java.util.Iterator;
12+
import java.util.LinkedHashSet;
13+
import java.util.List;
14+
import lombok.AccessLevel;
15+
import lombok.NoArgsConstructor;
16+
17+
/**
18+
* A validator for ensuring that export options are consistent with the ScalarDB table metadata and
19+
* follow the defined constraints.
20+
*/
21+
@NoArgsConstructor(access = AccessLevel.PRIVATE)
22+
public class ExportOptionsValidator {
23+
24+
/**
25+
* Validates the export request.
26+
*
27+
* @param exportOptions The export options provided by the user.
28+
* @param tableMetadata The metadata of the ScalarDB table to validate against.
29+
* @throws ExportOptionsValidationException If the export options are invalid.
30+
*/
31+
public static void validate(ExportOptions exportOptions, TableMetadata tableMetadata)
32+
throws ExportOptionsValidationException {
33+
LinkedHashSet<String> partitionKeyNames = tableMetadata.getPartitionKeyNames();
34+
LinkedHashSet<String> clusteringKeyNames = tableMetadata.getClusteringKeyNames();
35+
ScanRange scanRange = exportOptions.getScanRange();
36+
37+
validatePartitionKey(partitionKeyNames, exportOptions.getScanPartitionKey());
38+
validateProjectionColumns(tableMetadata.getColumnNames(), exportOptions.getProjectionColumns());
39+
validateSortOrders(clusteringKeyNames, exportOptions.getSortOrders());
40+
41+
if (scanRange.getScanStartKey() != null) {
42+
validateClusteringKey(clusteringKeyNames, scanRange.getScanStartKey());
43+
}
44+
if (scanRange.getScanEndKey() != null) {
45+
validateClusteringKey(clusteringKeyNames, scanRange.getScanEndKey());
46+
}
47+
}
48+
49+
/*
50+
* Check if the provided partition key is available in the ScalarDB table
51+
* @param partitionKeyNames List of partition key names available in a
52+
* @param key To be validated ScalarDB key
53+
* @throws ExportOptionsValidationException if the key could not be found or is not a partition
54+
*/
55+
private static void validatePartitionKey(LinkedHashSet<String> partitionKeyNames, Key key)
56+
throws ExportOptionsValidationException {
57+
if (partitionKeyNames == null || key == null) {
58+
return;
59+
}
60+
61+
// Make sure that all partition key columns are provided
62+
if (partitionKeyNames.size() != key.getColumns().size()) {
63+
throw new ExportOptionsValidationException(
64+
String.format(INCOMPLETE_PARTITION_KEY, partitionKeyNames));
65+
}
66+
67+
// Check if the order of columns in key.getColumns() matches the order in partitionKeyNames
68+
Iterator<String> partitionKeyIterator = partitionKeyNames.iterator();
69+
for (Column<?> column : key.getColumns()) {
70+
// Check if the column names match in order
71+
if (!partitionKeyIterator.hasNext()
72+
|| !partitionKeyIterator.next().equals(column.getName())) {
73+
throw new ExportOptionsValidationException(
74+
String.format(PARTITION_KEY_ORDER_MISMATCH, partitionKeyNames));
75+
}
76+
}
77+
}
78+
79+
private static void validateSortOrders(
80+
LinkedHashSet<String> clusteringKeyNames, List<Scan.Ordering> sortOrders)
81+
throws ExportOptionsValidationException {
82+
if (sortOrders == null || sortOrders.isEmpty()) {
83+
return;
84+
}
85+
86+
for (Scan.Ordering sortOrder : sortOrders) {
87+
checkIfColumnExistsAsClusteringKey(clusteringKeyNames, sortOrder.getColumnName());
88+
}
89+
}
90+
91+
/**
92+
* Validates that the clustering key columns in the given Key object match the expected order
93+
* defined in the clusteringKeyNames. The Key can be a prefix of the clusteringKeyNames, but the
94+
* order must remain consistent.
95+
*
96+
* @param clusteringKeyNames the expected ordered set of clustering key names
97+
* @param key the Key object containing the actual clustering key columns
98+
* @throws ExportOptionsValidationException if the order or names of clustering keys do not match
99+
*/
100+
private static void validateClusteringKey(LinkedHashSet<String> clusteringKeyNames, Key key)
101+
throws ExportOptionsValidationException {
102+
// If either clusteringKeyNames or key is null, no validation is needed
103+
if (clusteringKeyNames == null || key == null) {
104+
return;
105+
}
106+
107+
// Create an iterator to traverse the clusteringKeyNames in order
108+
Iterator<String> clusteringKeyIterator = clusteringKeyNames.iterator();
109+
110+
// Iterate through the columns in the given Key
111+
for (Column<?> column : key.getColumns()) {
112+
// If clusteringKeyNames have been exhausted but columns still exist in the Key,
113+
// it indicates a mismatch
114+
if (!clusteringKeyIterator.hasNext()) {
115+
throw new ExportOptionsValidationException(
116+
String.format(CLUSTERING_KEY_ORDER_MISMATCH, clusteringKeyNames));
117+
}
118+
119+
// Get the next expected clustering key name
120+
String expectedKey = clusteringKeyIterator.next();
121+
122+
// Check if the current column name matches the expected clustering key name
123+
if (!column.getName().equals(expectedKey)) {
124+
throw new ExportOptionsValidationException(
125+
String.format(CLUSTERING_KEY_ORDER_MISMATCH, clusteringKeyNames));
126+
}
127+
}
128+
}
129+
130+
private static void checkIfColumnExistsAsClusteringKey(
131+
LinkedHashSet<String> clusteringKeyNames, String columnName)
132+
throws ExportOptionsValidationException {
133+
if (clusteringKeyNames == null || columnName == null) {
134+
return;
135+
}
136+
137+
if (!clusteringKeyNames.contains(columnName)) {
138+
throw new ExportOptionsValidationException(
139+
String.format(CLUSTERING_KEY_NOT_FOUND, columnName));
140+
}
141+
}
142+
143+
private static void validateProjectionColumns(
144+
LinkedHashSet<String> columnNames, List<String> columns)
145+
throws ExportOptionsValidationException {
146+
if (columns == null || columns.isEmpty()) {
147+
return;
148+
}
149+
150+
for (String column : columns) {
151+
if (!columnNames.contains(column)) {
152+
throw new ExportOptionsValidationException(String.format(INVALID_PROJECTION, column));
153+
}
154+
}
155+
}
156+
}

‎data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/validation/ExportOptionsValidatorTest.java‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
package com.scalar.db.dataloader.core.dataexport.validation;
2+
3+
import static com.scalar.db.dataloader.core.ErrorMessage.*;
4+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
5+
6+
import com.scalar.db.api.TableMetadata;
7+
import com.scalar.db.dataloader.core.FileFormat;
8+
import com.scalar.db.dataloader.core.ScanRange;
9+
import com.scalar.db.dataloader.core.dataexport.ExportOptions;
10+
import com.scalar.db.io.DataType;
11+
import com.scalar.db.io.IntColumn;
12+
import com.scalar.db.io.Key;
13+
import com.scalar.db.io.TextColumn;
14+
import java.util.ArrayList;
15+
import java.util.Collections;
16+
import java.util.LinkedHashSet;
17+
import java.util.List;
18+
import org.junit.jupiter.api.BeforeEach;
19+
import org.junit.jupiter.api.Test;
20+
21+
class ExportOptionsValidatorTest {
22+
23+
private TableMetadata singlePkCkMetadata;
24+
private TableMetadata multiplePkCkMetadata;
25+
private List<String> projectedColumns;
26+
27+
@BeforeEach
28+
void setup() {
29+
singlePkCkMetadata = createMockMetadata(1, 1);
30+
multiplePkCkMetadata = createMockMetadata(2, 2);
31+
projectedColumns = createProjectedColumns();
32+
}
33+
34+
private TableMetadata createMockMetadata(int pkCount, int ckCount) {
35+
TableMetadata.Builder builder = TableMetadata.newBuilder();
36+
37+
// Add partition keys
38+
for (int i = 1; i <= pkCount; i++) {
39+
builder.addColumn("pk" + i, DataType.INT);
40+
builder.addPartitionKey("pk" + i);
41+
}
42+
43+
// Add clustering keys
44+
for (int i = 1; i <= ckCount; i++) {
45+
builder.addColumn("ck" + i, DataType.TEXT);
46+
builder.addClusteringKey("ck" + i);
47+
}
48+
49+
return builder.build();
50+
}
51+
52+
private List<String> createProjectedColumns() {
53+
List<String> columns = new ArrayList<>();
54+
columns.add("pk1");
55+
columns.add("ck1");
56+
return columns;
57+
}
58+
59+
@Test
60+
void validate_withValidExportOptionsForSinglePkCk_ShouldNotThrowException()
61+
throws ExportOptionsValidationException {
62+
63+
Key partitionKey = Key.newBuilder().add(IntColumn.of("pk1", 1)).build();
64+
65+
ExportOptions exportOptions =
66+
ExportOptions.builder("test", "sample", partitionKey, FileFormat.JSON)
67+
.projectionColumns(projectedColumns)
68+
.scanRange(new ScanRange(null, null, false, false))
69+
.build();
70+
71+
ExportOptionsValidator.validate(exportOptions, singlePkCkMetadata);
72+
}
73+
74+
@Test
75+
void validate_withValidExportOptionsForMultiplePkCk_ShouldNotThrowException()
76+
throws ExportOptionsValidationException {
77+
78+
Key partitionKey =
79+
Key.newBuilder().add(IntColumn.of("pk1", 1)).add(IntColumn.of("pk2", 2)).build();
80+
81+
ExportOptions exportOptions =
82+
ExportOptions.builder("test", "sample", partitionKey, FileFormat.JSON)
83+
.projectionColumns(projectedColumns)
84+
.scanRange(new ScanRange(null, null, false, false))
85+
.build();
86+
87+
ExportOptionsValidator.validate(exportOptions, multiplePkCkMetadata);
88+
}
89+
90+
@Test
91+
void validate_withIncompletePartitionKeyForSinglePk_ShouldThrowException() {
92+
Key incompletePartitionKey = Key.newBuilder().build();
93+
94+
ExportOptions exportOptions =
95+
ExportOptions.builder("test", "sample", incompletePartitionKey, FileFormat.JSON).build();
96+
97+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, singlePkCkMetadata))
98+
.isInstanceOf(ExportOptionsValidationException.class)
99+
.hasMessage(
100+
String.format(INCOMPLETE_PARTITION_KEY, singlePkCkMetadata.getPartitionKeyNames()));
101+
}
102+
103+
@Test
104+
void validate_withIncompletePartitionKeyForMultiplePks_ShouldThrowException() {
105+
Key incompletePartitionKey = Key.newBuilder().add(IntColumn.of("pk1", 1)).build();
106+
107+
ExportOptions exportOptions =
108+
ExportOptions.builder("test", "sample", incompletePartitionKey, FileFormat.JSON).build();
109+
110+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, multiplePkCkMetadata))
111+
.isInstanceOf(ExportOptionsValidationException.class)
112+
.hasMessage(
113+
String.format(INCOMPLETE_PARTITION_KEY, multiplePkCkMetadata.getPartitionKeyNames()));
114+
}
115+
116+
@Test
117+
void validate_withInvalidProjectionColumn_ShouldThrowException() {
118+
ExportOptions exportOptions =
119+
ExportOptions.builder(
120+
"test",
121+
"sample",
122+
Key.newBuilder().add(IntColumn.of("pk1", 1)).build(),
123+
FileFormat.JSON)
124+
.projectionColumns(Collections.singletonList("invalid_column"))
125+
.build();
126+
127+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, singlePkCkMetadata))
128+
.isInstanceOf(ExportOptionsValidationException.class)
129+
.hasMessage(String.format(INVALID_PROJECTION, "invalid_column"));
130+
}
131+
132+
@Test
133+
void validate_withInvalidClusteringKeyInScanRange_ShouldThrowException() {
134+
ScanRange scanRange =
135+
new ScanRange(
136+
Key.newBuilder().add(TextColumn.of("invalid_ck", "value")).build(),
137+
Key.newBuilder().add(TextColumn.of("ck1", "value")).build(),
138+
false,
139+
false);
140+
141+
ExportOptions exportOptions =
142+
ExportOptions.builder("test", "sample", createValidPartitionKey(), FileFormat.JSON)
143+
.scanRange(scanRange)
144+
.build();
145+
146+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, singlePkCkMetadata))
147+
.isInstanceOf(ExportOptionsValidationException.class)
148+
.hasMessage(String.format(CLUSTERING_KEY_ORDER_MISMATCH, "[ck1]"));
149+
}
150+
151+
@Test
152+
void validate_withInvalidPartitionKeyOrder_ShouldThrowException() {
153+
// Partition key names are expected to be "pk1", "pk2"
154+
LinkedHashSet<String> partitionKeyNames = new LinkedHashSet<>();
155+
partitionKeyNames.add("pk1");
156+
partitionKeyNames.add("pk2");
157+
158+
// Create a partition key with reversed order, expecting an error
159+
Key invalidPartitionKey =
160+
Key.newBuilder()
161+
.add(IntColumn.of("pk2", 2)) // Incorrect order
162+
.add(IntColumn.of("pk1", 1)) // Incorrect order
163+
.build();
164+
165+
ExportOptions exportOptions =
166+
ExportOptions.builder("test", "sample", invalidPartitionKey, FileFormat.JSON)
167+
.projectionColumns(projectedColumns)
168+
.scanRange(new ScanRange(null, null, false, false))
169+
.build();
170+
171+
// Verify that the validator throws the correct exception
172+
assertThatThrownBy(() -> ExportOptionsValidator.validate(exportOptions, multiplePkCkMetadata))
173+
.isInstanceOf(ExportOptionsValidationException.class)
174+
.hasMessage(String.format(PARTITION_KEY_ORDER_MISMATCH, partitionKeyNames));
175+
}
176+
177+
private Key createValidPartitionKey() {
178+
return Key.newBuilder().add(IntColumn.of("pk1", 1)).build();
179+
}
180+
}

0 commit comments

Comments
 (0)