Allow nulls in non-pk columns

Author: pravinbhat

pom.xml

@@ -8,7 +8,7 @@
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <revision>3.2.0</revision>
+    <revision>3.2.1</revision>
     <scala.version>2.12.17</scala.version>
     <scala.main.version>2.12</scala.main.version>
     <spark.version>3.3.1</spark.version>

src/main/java/datastax/astra/migrate/AbstractJobSession.java

@@ -254,11 +254,14 @@ private BoundStatement getBoundStatement(Row sourceRow, BoundStatement boundSele
         Object colData = getData(dataTypeObj, index, sourceRow);
 
         // Handle rows with blank values in primary-key fields
-        Optional<Object> optionalVal = handleBlankInPrimaryKey(index, colData, dataTypeObj.typeClass, sourceRow);
-        if (!optionalVal.isPresent()) {
-            return null;
+        if (index < idColTypes.size()) {
+            Optional<Object> optionalVal = handleBlankInPrimaryKey(index, colData, dataTypeObj.typeClass, sourceRow);
+            if (!optionalVal.isPresent()) {
+                return null;
+            }
+            colData = optionalVal.get();
         }
-        boundSelectStatement = boundSelectStatement.set(index, optionalVal.get(), dataTypeObj.typeClass);
+        boundSelectStatement = boundSelectStatement.set(index, colData, dataTypeObj.typeClass);
         return boundSelectStatement;
     }
 

src/resources/sparkConf.properties

@@ -64,7 +64,7 @@ spark.query.types                                 9,1,4,3
 #############################################################################################################
 
 # ENABLE ONLY IF COLUMN NAMES ON TARGET IS DIFFERENT FROM ORIGIN (SCHEMA & DATA-TYPES MUST BE SAME)
-#spark.query.target                                partition-key,clustering-key,order-date,amount
+#spark.query.target                                comma-separated-partition-key,comma-separated-clustering-key,comma-separated-other-columns
 
 # The tool adds TTL & Writetime at row-level (not field-level).
 # The largest TTL & Writetime values are used if multiple indexes are listed (comma separated)