Add Explicit \N NULL Marker for CSV Export/Import of TEXT Fields (#3199)

Co-authored-by: Peckstadt Yves <[email protected]>

Author: inv-jishnu

data-loader/core/src/main/java/com/scalar/db/dataloader/core/Constants.java

@@ -16,4 +16,12 @@ public class Constants {
    */
   public static final String ABORT_TRANSACTION_STATUS =
       "Transaction aborted as part of batch transaction aborted";
+  /**
+   * Special null value representation used for TEXT data type columns in CSV files.
+   *
+   * <p>This value is used to distinguish between an empty string and a null value in CSV exports
+   * and imports. When exporting, null TEXT values are converted to this string. When importing,
+   * this string is converted back to null for TEXT columns.
+   */
+  public static final String CSV_TEXT_NULL_VALUE = "\\N";
 }

data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTask.java

@@ -2,6 +2,7 @@
 
 import com.scalar.db.api.Result;
 import com.scalar.db.api.TableMetadata;
+import com.scalar.db.dataloader.core.Constants;
 import com.scalar.db.dataloader.core.DataLoaderError;
 import com.scalar.db.dataloader.core.util.CsvUtil;
 import com.scalar.db.dataloader.core.util.DecimalUtil;
@@ -118,6 +119,11 @@ private String convertResultToCsv(Result result) {
    */
   private String convertToString(Result result, String columnName, DataType dataType) {
     if (result.isNull(columnName)) {
+      // Special null value is added when a column of text data type has null value. This is only
+      // converted for CSV files
+      if (dataType.equals(DataType.TEXT)) {
+        return Constants.CSV_TEXT_NULL_VALUE;
+      }
       return null;
     }
     String value = "";

data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java

@@ -4,6 +4,7 @@
 import com.scalar.db.api.Result;
 import com.scalar.db.api.TableMetadata;
 import com.scalar.db.dataloader.core.ColumnInfo;
+import com.scalar.db.dataloader.core.Constants;
 import com.scalar.db.dataloader.core.DataLoaderError;
 import com.scalar.db.dataloader.core.exception.Base64Exception;
 import com.scalar.db.dataloader.core.exception.ColumnParsingException;
@@ -83,8 +84,14 @@ public static Column<?> createColumnFromValue(
       DataType dataType, ColumnInfo columnInfo, @Nullable String value)
       throws ColumnParsingException {
     String columnName = columnInfo.getColumnName();
-    if (value != null && !dataType.equals(DataType.TEXT) && value.equalsIgnoreCase("null")) {
-      value = null;
+    if (value != null) {
+      if (dataType.equals(DataType.TEXT)) {
+        if (Constants.CSV_TEXT_NULL_VALUE.equals(value)) {
+          value = null;
+        }
+      } else if (value.equalsIgnoreCase("null")) {
+        value = null;
+      }
     }
     try {
       switch (dataType) {

data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTaskTest.java

@@ -3,9 +3,11 @@
 import com.scalar.db.api.Result;
 import com.scalar.db.api.TableMetadata;
 import com.scalar.db.common.ResultImpl;
+import com.scalar.db.dataloader.core.Constants;
 import com.scalar.db.dataloader.core.UnitTestUtils;
 import com.scalar.db.io.Column;
 import com.scalar.db.io.DataType;
+import com.scalar.db.io.TextColumn;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -116,4 +118,31 @@ void process_withValidResultList_withPartialProjectionsAndMetadata_shouldReturnV
     String output = csvProducerTask.process(resultList);
     Assertions.assertEquals(expectedOutput, output.trim());
   }
+
+  @Test
+  void
+      process_withValidResultListWithTextFieldWithNullValue_shouldReturnValidCsvStringWithCustomNullValueForTextField() {
+    String expectedOutput =
+        "9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,"
+            + Constants.CSV_TEXT_NULL_VALUE
+            + ",YmxvYiB0ZXN0IHZhbHVl,2000-01-01,01:01:01,2000-01-01T01:01,1970-01-21T03:20:41.740Z,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,txt value 464654654,2147483647,2147483647,9007199254740992,9007199254740992,test value,2147483647,2147483647,9007199254740992,9007199254740992";
+    Map<String, Column<?>> values = UnitTestUtils.createTestValues();
+    String textColName = "col6";
+    Column<?> col = values.get(textColName);
+    if (col instanceof TextColumn) {
+      values.put(textColName, TextColumn.of(textColName, null));
+    }
+    Result result = new ResultImpl(values, mockMetadata);
+    List<Result> resultList = new ArrayList<>();
+    resultList.add(result);
+    String output = csvProducerTask.process(resultList);
+    Assertions.assertEquals(expectedOutput, output.trim());
+
+    values.put(textColName, TextColumn.ofNull(textColName));
+    result = new ResultImpl(values, mockMetadata);
+    resultList = new ArrayList<>();
+    resultList.add(result);
+    output = csvProducerTask.process(resultList);
+    Assertions.assertEquals(expectedOutput, output.trim());
+  }
 }

data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java

@@ -8,6 +8,7 @@
 import com.scalar.db.api.TableMetadata;
 import com.scalar.db.common.ResultImpl;
 import com.scalar.db.dataloader.core.ColumnInfo;
+import com.scalar.db.dataloader.core.Constants;
 import com.scalar.db.dataloader.core.DataLoaderError;
 import com.scalar.db.dataloader.core.UnitTestUtils;
 import com.scalar.db.dataloader.core.exception.Base64Exception;
@@ -309,4 +310,19 @@ void createColumnFromValue_valueIsNullString_shouldRemainLiteralForTextType()
     textCol = ColumnUtils.createColumnFromValue(DataType.TEXT, columnInfo, "nuLL");
     assertEquals(TextColumn.of(columnName, "nuLL"), textCol);
   }
+
+  /**
+   * Tests that when the string value has custom null value is provided for TEXT columns, it is
+   * treated as an actual null value
+   */
+  @Test
+  void createColumnFromValue_customNullValueForText_shouldBeConvertedToNull()
+      throws ColumnParsingException {
+    String columnName = "textColumn";
+    ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build();
+
+    Column<?> textCol =
+        ColumnUtils.createColumnFromValue(DataType.TEXT, columnInfo, Constants.CSV_TEXT_NULL_VALUE);
+    assertEquals(TextColumn.ofNull(columnName), textCol);
+  }
 }