Skip to content

Resolve minor bugs in data loader core #2752

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 12, 2025
Merged

Resolve minor bugs in data loader core #2752

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4c34a12
Bug fixes
inv-jishnu Jun 10, 2025
8ad7213
Merge branch 'master' into feat/data-loader/core-minor-fixes
inv-jishnu Jun 10, 2025
aa31b10
Merge branch 'master' into feat/data-loader/core-minor-fixes
inv-jishnu Jun 11, 2025
9c4acac
Merge branch 'master' into feat/data-loader/core-minor-fixes
inv-jishnu Jun 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions core/src/main/java/com/scalar/db/common/error/CoreError.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -689,13 +689,13 @@ public enum CoreError implements ScalarDbError {
DATA_LOADER_INVALID_BASE64_ENCODING_FOR_COLUMN_VALUE(
Category.USER_ERROR,
"0149",
"Invalid base64 encoding for blob value for column %s in table %s in namespace %s",
"Invalid base64 encoding for blob value '%s' for column %s in table %s in namespace %s",
Copy link

Copilot AI Jun 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nitpick] Add a trailing period to this error message string to match the punctuation style of other CoreError templates (e.g., end with a period).

Suggested change
"Invalid base64 encoding for blob value '%s' for column %s in table %s in namespace %s",
"Invalid base64 encoding for blob value '%s' for column %s in table %s in namespace %s.",

Copilot uses AI. Check for mistakes.
"",
""),
DATA_LOADER_INVALID_NUMBER_FORMAT_FOR_COLUMN_VALUE(
Category.USER_ERROR,
"0150",
"Invalid number specified for column %s in table %s in namespace %s",
"Invalid number '%s' specified for column %s in table %s in namespace %s",
"",
""),
DATA_LOADER_ERROR_METHOD_NULL_ARGUMENT(
Expand Down Expand Up @@ -898,7 +898,7 @@ public enum CoreError implements ScalarDbError {
DATA_LOADER_INVALID_DATE_TIME_FOR_COLUMN_VALUE(
Category.USER_ERROR,
"0199",
"Invalid date time value specified for column %s in table %s in namespace %s.",
"Invalid date time value '%s' specified for column %s in table %s in namespace %s.",
"",
""),
DATA_LOADER_NULL_OR_EMPTY_KEY_VALUE_INPUT(
Expand Down
18 changes: 5 additions & 13 deletions ...ore/src/main/java/com/scalar/db/dataloader/core/dataimport/processor/ImportProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,19 +416,11 @@ private ImportDataChunkStatus processDataChunkWithTransactions(
ImportTransactionBatchResult importTransactionBatchResult =
processTransactionBatch(dataChunk.getDataChunkId(), transactionBatch);

importTransactionBatchResult
.getRecords()
.forEach(
batchRecords -> {
if (batchRecords.getTargets().stream()
.allMatch(
targetResult ->
targetResult.getStatus().equals(ImportTargetResultStatus.SAVED))) {
successCount.incrementAndGet();
} else {
failureCount.incrementAndGet();
}
});
if (importTransactionBatchResult.isSuccess()) {
successCount.addAndGet(importTransactionBatchResult.getRecords().size());
} else {
failureCount.addAndGet(importTransactionBatchResult.getRecords().size());
}
}
Instant endTime = Instant.now();
int totalDuration = (int) Duration.between(startTime, endTime).toMillis();
Expand Down
7 changes: 5 additions & 2 deletions data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public ImportTaskResult execute() {
return ImportTaskResult.builder()
.rawRecord(params.getSourceRecord())
.rowNumber(params.getRowNumber())
.targets(Collections.singletonList(singleTargetResult))
.targets(new ArrayList<>(Collections.singletonList(singleTargetResult)))
.build();
}

Expand Down Expand Up @@ -308,11 +308,14 @@ && shouldRevalidateMissingColumns(importOptions, checkForMissingColumns)) {
optionalScalarDBResult.orElse(null),
mutableSourceRecord,
importOptions.isIgnoreNullValues(),
tableMetadata);
tableMetadata,
namespace,
table);
} catch (Base64Exception | ColumnParsingException e) {
return ImportTargetResult.builder()
.namespace(namespace)
.tableName(table)
.importedRecord(mutableSourceRecord)
.status(ImportTargetResultStatus.VALIDATION_FAILED)
.errors(Collections.singletonList(e.getMessage()))
.build();
Expand Down
33 changes: 24 additions & 9 deletions data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,17 +134,17 @@ public static Column<?> createColumnFromValue(
} catch (NumberFormatException e) {
throw new ColumnParsingException(
CoreError.DATA_LOADER_INVALID_NUMBER_FORMAT_FOR_COLUMN_VALUE.buildMessage(
columnName, columnInfo.getTableName(), columnInfo.getNamespace()),
value, columnName, columnInfo.getTableName(), columnInfo.getNamespace()),
e);
} catch (DateTimeParseException e) {
throw new ColumnParsingException(
CoreError.DATA_LOADER_INVALID_DATE_TIME_FOR_COLUMN_VALUE.buildMessage(
columnName, columnInfo.getTableName(), columnInfo.getNamespace()),
value, columnName, columnInfo.getTableName(), columnInfo.getNamespace()),
e);
} catch (IllegalArgumentException e) {
throw new ColumnParsingException(
CoreError.DATA_LOADER_INVALID_BASE64_ENCODING_FOR_COLUMN_VALUE.buildMessage(
columnName, columnInfo.getTableName(), columnInfo.getNamespace()),
value, columnName, columnInfo.getTableName(), columnInfo.getNamespace()),
e);
}
}
Expand All @@ -166,6 +166,8 @@ public static Column<?> createColumnFromValue(
* @param sourceRecord the source data in JSON format to compare against
* @param ignoreNullValues if true, null values will be excluded from the result
* @param tableMetadata metadata about the table structure and column types
* @param namespace namespace in which the table is present
* @param table table name to which data is to be imported
* @return a List of Column objects representing the processed data
* @throws Base64Exception if there's an error processing base64 encoded BLOB data
* @throws ColumnParsingException if there's an error parsing column values
Expand All @@ -174,7 +176,9 @@ public static List<Column<?>> getColumnsFromResult(
Result scalarDBResult,
JsonNode sourceRecord,
boolean ignoreNullValues,
TableMetadata tableMetadata)
TableMetadata tableMetadata,
String namespace,
String table)
throws Base64Exception, ColumnParsingException {

List<Column<?>> columns = new ArrayList<>();
Expand All @@ -193,7 +197,9 @@ public static List<Column<?>> getColumnsFromResult(
sourceRecord,
columnName,
ignoreNullValues,
tableMetadata.getColumnDataTypes());
tableMetadata.getColumnDataTypes(),
namespace,
table);

if (column != null) {
columns.add(column);
Expand Down Expand Up @@ -242,6 +248,8 @@ private static Set<String> getColumnsToIgnore(
* @param columnName the name of the column to retrieve
* @param ignoreNullValues whether to ignore null values in the result
* @param dataTypesByColumns mapping of column names to their data types
* @param namespace namespace in which the table is present
* @param table table name to which data is to be imported
* @return the Column object containing the value, or null if ignored
* @throws ColumnParsingException if there's an error parsing the column value
*/
Expand All @@ -250,13 +258,15 @@ private static Column<?> getColumn(
JsonNode sourceRecord,
String columnName,
boolean ignoreNullValues,
Map<String, DataType> dataTypesByColumns)
Map<String, DataType> dataTypesByColumns,
String namespace,
String table)
throws ColumnParsingException {
if (scalarDBResult != null && !sourceRecord.has(columnName)) {
return getColumnFromResult(scalarDBResult, columnName);
} else {
return getColumnFromSourceRecord(
sourceRecord, columnName, ignoreNullValues, dataTypesByColumns);
sourceRecord, columnName, ignoreNullValues, dataTypesByColumns, namespace, table);
}
}

Expand All @@ -279,22 +289,27 @@ private static Column<?> getColumnFromResult(Result scalarDBResult, String colum
* @param columnName column name
* @param ignoreNullValues ignore null values or not
* @param dataTypesByColumns data types of columns
* @param namespace namespace in which the table is present
* @param table table name to which data is to be imported
* @return column data
* @throws ColumnParsingException if an error occurs while parsing the column
*/
private static Column<?> getColumnFromSourceRecord(
JsonNode sourceRecord,
String columnName,
boolean ignoreNullValues,
Map<String, DataType> dataTypesByColumns)
Map<String, DataType> dataTypesByColumns,
String namespace,
String table)
throws ColumnParsingException {
DataType dataType = dataTypesByColumns.get(columnName);
String columnValue =
sourceRecord.has(columnName) && !sourceRecord.get(columnName).isNull()
? sourceRecord.get(columnName).asText()
: null;
if (!ignoreNullValues || columnValue != null) {
ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build();
ColumnInfo columnInfo =
ColumnInfo.builder().columnName(columnName).tableName(table).namespace(namespace).build();
return createColumnFromValue(dataType, columnInfo, columnValue);
}
return null;
Expand Down
18 changes: 14 additions & 4 deletions data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ void createColumnFromValue_invalidNumberFormat_throwsNumberFormatException() {
() -> ColumnUtils.createColumnFromValue(DataType.INT, columnInfo, value));
assertEquals(
CoreError.DATA_LOADER_INVALID_NUMBER_FORMAT_FOR_COLUMN_VALUE.buildMessage(
columnName, "table", "ns"),
value, columnName, "table", "ns"),
exception.getMessage());
}

Expand All @@ -181,7 +181,7 @@ void createColumnFromValue_invalidBase64_throwsBase64Exception() {
() -> ColumnUtils.createColumnFromValue(DataType.BLOB, columnInfo, value));
assertEquals(
CoreError.DATA_LOADER_INVALID_BASE64_ENCODING_FOR_COLUMN_VALUE.buildMessage(
columnName, "table", "ns"),
value, columnName, "table", "ns"),
exception.getMessage());
}
/**
Expand All @@ -200,7 +200,7 @@ void createColumnFromValue_invalidDateTimeFormat_throwsDateTimeParseException()
() -> ColumnUtils.createColumnFromValue(DataType.TIMESTAMP, columnInfo, value));
assertEquals(
CoreError.DATA_LOADER_INVALID_DATE_TIME_FOR_COLUMN_VALUE.buildMessage(
columnName, "table", "ns"),
value, columnName, "table", "ns"),
exception.getMessage());
}

Expand All @@ -215,7 +215,17 @@ void createColumnFromValue_invalidDateTimeFormat_throwsDateTimeParseException()
void getColumnsFromResult_withValidData_shouldReturnColumns()
throws Base64Exception, ColumnParsingException {
List<Column<?>> columns =
ColumnUtils.getColumnsFromResult(scalarDBResult, sourceRecord, false, mockMetadata);
ColumnUtils.getColumnsFromResult(
scalarDBResult, sourceRecord, false, mockMetadata, "namespace", "table");
assertEquals(8, columns.size());
}

@Test
void getColumnsFromResult_withResultNull_withValidData_shouldReturnColumns()
throws Base64Exception, ColumnParsingException {
List<Column<?>> columns =
ColumnUtils.getColumnsFromResult(
null, sourceRecord, false, mockMetadata, "namespace", "table");
assertEquals(8, columns.size());
}
}