From 4c34a12a3027ffcb705a1b60886cbcb779d966d7 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Tue, 10 Jun 2025 17:10:04 +0530 Subject: [PATCH] Bug fixes --- .../com/scalar/db/common/error/CoreError.java | 6 ++-- .../dataimport/processor/ImportProcessor.java | 18 +++------- .../core/dataimport/task/ImportTask.java | 7 ++-- .../db/dataloader/core/util/ColumnUtils.java | 33 ++++++++++++++----- .../dataloader/core/util/ColumnUtilsTest.java | 18 +++++++--- 5 files changed, 51 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/com/scalar/db/common/error/CoreError.java b/core/src/main/java/com/scalar/db/common/error/CoreError.java index b6e450107e..5aae66ab5c 100644 --- a/core/src/main/java/com/scalar/db/common/error/CoreError.java +++ b/core/src/main/java/com/scalar/db/common/error/CoreError.java @@ -689,13 +689,13 @@ public enum CoreError implements ScalarDbError { DATA_LOADER_INVALID_BASE64_ENCODING_FOR_COLUMN_VALUE( Category.USER_ERROR, "0149", - "Invalid base64 encoding for blob value for column %s in table %s in namespace %s", + "Invalid base64 encoding for blob value '%s' for column %s in table %s in namespace %s", "", ""), DATA_LOADER_INVALID_NUMBER_FORMAT_FOR_COLUMN_VALUE( Category.USER_ERROR, "0150", - "Invalid number specified for column %s in table %s in namespace %s", + "Invalid number '%s' specified for column %s in table %s in namespace %s", "", ""), DATA_LOADER_ERROR_METHOD_NULL_ARGUMENT( @@ -899,7 +899,7 @@ public enum CoreError implements ScalarDbError { DATA_LOADER_INVALID_DATE_TIME_FOR_COLUMN_VALUE( Category.USER_ERROR, "0199", - "Invalid date time value specified for column %s in table %s in namespace %s.", + "Invalid date time value '%s' specified for column %s in table %s in namespace %s.", "", ""), DATA_LOADER_NULL_OR_EMPTY_KEY_VALUE_INPUT( diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/processor/ImportProcessor.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/processor/ImportProcessor.java index 3f191f7259..c11ae3f655 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/processor/ImportProcessor.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/processor/ImportProcessor.java @@ -416,19 +416,11 @@ private ImportDataChunkStatus processDataChunkWithTransactions( ImportTransactionBatchResult importTransactionBatchResult = processTransactionBatch(dataChunk.getDataChunkId(), transactionBatch); - importTransactionBatchResult - .getRecords() - .forEach( - batchRecords -> { - if (batchRecords.getTargets().stream() - .allMatch( - targetResult -> - targetResult.getStatus().equals(ImportTargetResultStatus.SAVED))) { - successCount.incrementAndGet(); - } else { - failureCount.incrementAndGet(); - } - }); + if (importTransactionBatchResult.isSuccess()) { + successCount.addAndGet(importTransactionBatchResult.getRecords().size()); + } else { + failureCount.addAndGet(importTransactionBatchResult.getRecords().size()); + } } Instant endTime = Instant.now(); int totalDuration = (int) Duration.between(startTime, endTime).toMillis(); diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTask.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTask.java index 26b4993977..5e64e4d63d 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTask.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTask.java @@ -73,7 +73,7 @@ public ImportTaskResult execute() { return ImportTaskResult.builder() .rawRecord(params.getSourceRecord()) .rowNumber(params.getRowNumber()) - .targets(Collections.singletonList(singleTargetResult)) + .targets(new ArrayList<>(Collections.singletonList(singleTargetResult))) .build(); } @@ -308,11 +308,14 @@ && shouldRevalidateMissingColumns(importOptions, checkForMissingColumns)) { optionalScalarDBResult.orElse(null), mutableSourceRecord, importOptions.isIgnoreNullValues(), - tableMetadata); + tableMetadata, + namespace, + table); } catch (Base64Exception | ColumnParsingException e) { return ImportTargetResult.builder() .namespace(namespace) .tableName(table) + .importedRecord(mutableSourceRecord) .status(ImportTargetResultStatus.VALIDATION_FAILED) .errors(Collections.singletonList(e.getMessage())) .build(); diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java index d6a653cced..90001ed062 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java @@ -134,17 +134,17 @@ public static Column createColumnFromValue( } catch (NumberFormatException e) { throw new ColumnParsingException( CoreError.DATA_LOADER_INVALID_NUMBER_FORMAT_FOR_COLUMN_VALUE.buildMessage( - columnName, columnInfo.getTableName(), columnInfo.getNamespace()), + value, columnName, columnInfo.getTableName(), columnInfo.getNamespace()), e); } catch (DateTimeParseException e) { throw new ColumnParsingException( CoreError.DATA_LOADER_INVALID_DATE_TIME_FOR_COLUMN_VALUE.buildMessage( - columnName, columnInfo.getTableName(), columnInfo.getNamespace()), + value, columnName, columnInfo.getTableName(), columnInfo.getNamespace()), e); } catch (IllegalArgumentException e) { throw new ColumnParsingException( CoreError.DATA_LOADER_INVALID_BASE64_ENCODING_FOR_COLUMN_VALUE.buildMessage( - columnName, columnInfo.getTableName(), columnInfo.getNamespace()), + value, columnName, columnInfo.getTableName(), columnInfo.getNamespace()), e); } } @@ -166,6 +166,8 @@ public static Column createColumnFromValue( * @param sourceRecord the source data in JSON format to compare against * @param ignoreNullValues if true, null values will be excluded from the result * @param tableMetadata metadata about the table structure and column types + * @param namespace namespace in which the table is present + * @param table table name to which data is to be imported * @return a List of Column objects representing the processed data * @throws Base64Exception if there's an error processing base64 encoded BLOB data * @throws ColumnParsingException if there's an error parsing column values @@ -174,7 +176,9 @@ public static List> getColumnsFromResult( Result scalarDBResult, JsonNode sourceRecord, boolean ignoreNullValues, - TableMetadata tableMetadata) + TableMetadata tableMetadata, + String namespace, + String table) throws Base64Exception, ColumnParsingException { List> columns = new ArrayList<>(); @@ -193,7 +197,9 @@ public static List> getColumnsFromResult( sourceRecord, columnName, ignoreNullValues, - tableMetadata.getColumnDataTypes()); + tableMetadata.getColumnDataTypes(), + namespace, + table); if (column != null) { columns.add(column); @@ -242,6 +248,8 @@ private static Set getColumnsToIgnore( * @param columnName the name of the column to retrieve * @param ignoreNullValues whether to ignore null values in the result * @param dataTypesByColumns mapping of column names to their data types + * @param namespace namespace in which the table is present + * @param table table name to which data is to be imported * @return the Column object containing the value, or null if ignored * @throws ColumnParsingException if there's an error parsing the column value */ @@ -250,13 +258,15 @@ private static Column getColumn( JsonNode sourceRecord, String columnName, boolean ignoreNullValues, - Map dataTypesByColumns) + Map dataTypesByColumns, + String namespace, + String table) throws ColumnParsingException { if (scalarDBResult != null && !sourceRecord.has(columnName)) { return getColumnFromResult(scalarDBResult, columnName); } else { return getColumnFromSourceRecord( - sourceRecord, columnName, ignoreNullValues, dataTypesByColumns); + sourceRecord, columnName, ignoreNullValues, dataTypesByColumns, namespace, table); } } @@ -279,6 +289,8 @@ private static Column getColumnFromResult(Result scalarDBResult, String colum * @param columnName column name * @param ignoreNullValues ignore null values or not * @param dataTypesByColumns data types of columns + * @param namespace namespace in which the table is present + * @param table table name to which data is to be imported * @return column data * @throws ColumnParsingException if an error occurs while parsing the column */ @@ -286,7 +298,9 @@ private static Column getColumnFromSourceRecord( JsonNode sourceRecord, String columnName, boolean ignoreNullValues, - Map dataTypesByColumns) + Map dataTypesByColumns, + String namespace, + String table) throws ColumnParsingException { DataType dataType = dataTypesByColumns.get(columnName); String columnValue = @@ -294,7 +308,8 @@ private static Column getColumnFromSourceRecord( ? sourceRecord.get(columnName).asText() : null; if (!ignoreNullValues || columnValue != null) { - ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build(); + ColumnInfo columnInfo = + ColumnInfo.builder().columnName(columnName).tableName(table).namespace(namespace).build(); return createColumnFromValue(dataType, columnInfo, columnValue); } return null; diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java index 49616cb02e..2ecd782fa6 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java @@ -161,7 +161,7 @@ void createColumnFromValue_invalidNumberFormat_throwsNumberFormatException() { () -> ColumnUtils.createColumnFromValue(DataType.INT, columnInfo, value)); assertEquals( CoreError.DATA_LOADER_INVALID_NUMBER_FORMAT_FOR_COLUMN_VALUE.buildMessage( - columnName, "table", "ns"), + value, columnName, "table", "ns"), exception.getMessage()); } @@ -181,7 +181,7 @@ void createColumnFromValue_invalidBase64_throwsBase64Exception() { () -> ColumnUtils.createColumnFromValue(DataType.BLOB, columnInfo, value)); assertEquals( CoreError.DATA_LOADER_INVALID_BASE64_ENCODING_FOR_COLUMN_VALUE.buildMessage( - columnName, "table", "ns"), + value, columnName, "table", "ns"), exception.getMessage()); } /** @@ -200,7 +200,7 @@ void createColumnFromValue_invalidDateTimeFormat_throwsDateTimeParseException() () -> ColumnUtils.createColumnFromValue(DataType.TIMESTAMP, columnInfo, value)); assertEquals( CoreError.DATA_LOADER_INVALID_DATE_TIME_FOR_COLUMN_VALUE.buildMessage( - columnName, "table", "ns"), + value, columnName, "table", "ns"), exception.getMessage()); } @@ -215,7 +215,17 @@ void createColumnFromValue_invalidDateTimeFormat_throwsDateTimeParseException() void getColumnsFromResult_withValidData_shouldReturnColumns() throws Base64Exception, ColumnParsingException { List> columns = - ColumnUtils.getColumnsFromResult(scalarDBResult, sourceRecord, false, mockMetadata); + ColumnUtils.getColumnsFromResult( + scalarDBResult, sourceRecord, false, mockMetadata, "namespace", "table"); + assertEquals(8, columns.size()); + } + + @Test + void getColumnsFromResult_withResultNull_withValidData_shouldReturnColumns() + throws Base64Exception, ColumnParsingException { + List> columns = + ColumnUtils.getColumnsFromResult( + null, sourceRecord, false, mockMetadata, "namespace", "table"); assertEquals(8, columns.size()); } }