added config for 10MB value

Author: mfmaher2

src/main/java/datastax/astra/migrate/OriginCountJobSession.java

@@ -58,7 +58,7 @@ protected OriginCountJobSession(CqlSession sourceSession, SparkConf sparkConf) {
         filterColName = sparkConf.get("spark.origin.FilterColumn");
         filterColType = sparkConf.get("spark.origin.FilterColumnType");
         filterColIndex =  Integer.parseInt(sparkConf.get("spark.origin.FilterColumnIndex", "0"));
-        fieldGuardraillimitMB =  Integer.parseInt(sparkConf.get("spark.fieldGuardraillimitMB", "0"));
+        fieldGuardraillimitMB =   Integer.parseInt(sparkConf.get("spark.fieldGuardraillimitMB", "0"));
 
         String partionKey = sparkConf.get("spark.query.cols.partitionKey");
         idColTypes = getTypes(sparkConf.get("spark.query.cols.id.types"));
@@ -94,7 +94,7 @@ public void getData(BigInteger min, BigInteger max) {
                         if(checkTableforColSize) {
                             int rowColcnt = GetRowColumnLength(sourceRow, filterColType, filterColIndex);
                             String result = "";
-                            if (rowColcnt > fieldGuardraillimitMB) {
+                            if (rowColcnt > fieldGuardraillimitMB * 1048576) {
                                 for (int index = 0; index < checkTableforColSizeTypes.size(); index++) {
                                     MigrateDataType dataType = checkTableforColSizeTypes.get(index);
                                     Object colData = getData(dataType, index, sourceRow);
@@ -116,7 +116,7 @@ public void getData(BigInteger min, BigInteger max) {
                         if(checkTableforColSize) {
                             int rowColcnt = GetRowColumnLength(sourceRow, filterColType, filterColIndex);
                             String result = "";
-                            if (rowColcnt > fieldGuardraillimitMB) {
+                            if (rowColcnt > fieldGuardraillimitMB * 1048576) {
                                 for (int index = 0; index < checkTableforColSizeTypes.size(); index++) {
                                     MigrateDataType dataType = checkTableforColSizeTypes.get(index);
                                     Object colData = getData(dataType, index, sourceRow);

src/resources/sparkConf.properties

@@ -28,7 +28,7 @@ spark.splitSize                                   10000
 spark.batchSize                                   5
 spark.coveragePercent                             100
 spark.printStatsAfter                             100000
-spark.fieldGuardraillimitMB                       10485760
+spark.fieldGuardraillimitMB                       10
 
 spark.query.origin                                partition-key,clustering-key,order-date,amount
 spark.query.origin.partitionKey                   partition-key