diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/Constants.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/Constants.java index 0d5996747a..b524cedb7e 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/Constants.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/Constants.java @@ -16,4 +16,12 @@ public class Constants { */ public static final String ABORT_TRANSACTION_STATUS = "Transaction aborted as part of batch transaction aborted"; + /** + * Special null value representation used for TEXT data type columns in CSV files. + * + *

This value is used to distinguish between an empty string and a null value in CSV exports + * and imports. When exporting, null TEXT values are converted to this string. When importing, + * this string is converted back to null for TEXT columns. + */ + public static final String CSV_TEXT_NULL_VALUE = "\\N"; } diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTask.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTask.java index fe48106b29..5a46a53227 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTask.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTask.java @@ -2,6 +2,7 @@ import com.scalar.db.api.Result; import com.scalar.db.api.TableMetadata; +import com.scalar.db.dataloader.core.Constants; import com.scalar.db.dataloader.core.DataLoaderError; import com.scalar.db.dataloader.core.util.CsvUtil; import com.scalar.db.dataloader.core.util.DecimalUtil; @@ -118,6 +119,11 @@ private String convertResultToCsv(Result result) { */ private String convertToString(Result result, String columnName, DataType dataType) { if (result.isNull(columnName)) { + // Special null value is added when a column of text data type has null value. This is only + // converted for CSV files + if (dataType.equals(DataType.TEXT)) { + return Constants.CSV_TEXT_NULL_VALUE; + } return null; } String value = ""; diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java index 40bd92d00e..6c981dd0af 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java @@ -4,6 +4,7 @@ import com.scalar.db.api.Result; import com.scalar.db.api.TableMetadata; import com.scalar.db.dataloader.core.ColumnInfo; +import com.scalar.db.dataloader.core.Constants; import com.scalar.db.dataloader.core.DataLoaderError; import com.scalar.db.dataloader.core.exception.Base64Exception; import com.scalar.db.dataloader.core.exception.ColumnParsingException; @@ -83,8 +84,14 @@ public static Column createColumnFromValue( DataType dataType, ColumnInfo columnInfo, @Nullable String value) throws ColumnParsingException { String columnName = columnInfo.getColumnName(); - if (value != null && !dataType.equals(DataType.TEXT) && value.equalsIgnoreCase("null")) { - value = null; + if (value != null) { + if (dataType.equals(DataType.TEXT)) { + if (Constants.CSV_TEXT_NULL_VALUE.equals(value)) { + value = null; + } + } else if (value.equalsIgnoreCase("null")) { + value = null; + } } try { switch (dataType) { diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTaskTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTaskTest.java index 2e0a7f334a..3e6005b28f 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTaskTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTaskTest.java @@ -3,9 +3,11 @@ import com.scalar.db.api.Result; import com.scalar.db.api.TableMetadata; import com.scalar.db.common.ResultImpl; +import com.scalar.db.dataloader.core.Constants; import com.scalar.db.dataloader.core.UnitTestUtils; import com.scalar.db.io.Column; import com.scalar.db.io.DataType; +import com.scalar.db.io.TextColumn; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -116,4 +118,31 @@ void process_withValidResultList_withPartialProjectionsAndMetadata_shouldReturnV String output = csvProducerTask.process(resultList); Assertions.assertEquals(expectedOutput, output.trim()); } + + @Test + void + process_withValidResultListWithTextFieldWithNullValue_shouldReturnValidCsvStringWithCustomNullValueForTextField() { + String expectedOutput = + "9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049," + + Constants.CSV_TEXT_NULL_VALUE + + ",YmxvYiB0ZXN0IHZhbHVl,2000-01-01,01:01:01,2000-01-01T01:01,1970-01-21T03:20:41.740Z,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,txt value 464654654,2147483647,2147483647,9007199254740992,9007199254740992,test value,2147483647,2147483647,9007199254740992,9007199254740992"; + Map> values = UnitTestUtils.createTestValues(); + String textColName = "col6"; + Column col = values.get(textColName); + if (col instanceof TextColumn) { + values.put(textColName, TextColumn.of(textColName, null)); + } + Result result = new ResultImpl(values, mockMetadata); + List resultList = new ArrayList<>(); + resultList.add(result); + String output = csvProducerTask.process(resultList); + Assertions.assertEquals(expectedOutput, output.trim()); + + values.put(textColName, TextColumn.ofNull(textColName)); + result = new ResultImpl(values, mockMetadata); + resultList = new ArrayList<>(); + resultList.add(result); + output = csvProducerTask.process(resultList); + Assertions.assertEquals(expectedOutput, output.trim()); + } } diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java index 1f00c04cd5..b4296e5978 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java @@ -8,6 +8,7 @@ import com.scalar.db.api.TableMetadata; import com.scalar.db.common.ResultImpl; import com.scalar.db.dataloader.core.ColumnInfo; +import com.scalar.db.dataloader.core.Constants; import com.scalar.db.dataloader.core.DataLoaderError; import com.scalar.db.dataloader.core.UnitTestUtils; import com.scalar.db.dataloader.core.exception.Base64Exception; @@ -309,4 +310,19 @@ void createColumnFromValue_valueIsNullString_shouldRemainLiteralForTextType() textCol = ColumnUtils.createColumnFromValue(DataType.TEXT, columnInfo, "nuLL"); assertEquals(TextColumn.of(columnName, "nuLL"), textCol); } + + /** + * Tests that when the string value has custom null value is provided for TEXT columns, it is + * treated as an actual null value + */ + @Test + void createColumnFromValue_customNullValueForText_shouldBeConvertedToNull() + throws ColumnParsingException { + String columnName = "textColumn"; + ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build(); + + Column textCol = + ColumnUtils.createColumnFromValue(DataType.TEXT, columnInfo, Constants.CSV_TEXT_NULL_VALUE); + assertEquals(TextColumn.ofNull(columnName), textCol); + } }