CDM-88: Added/updated SIT feature tests

Author: pravinbhat

README.md

@@ -34,7 +34,7 @@ tar -xvzf spark-3.3.1-bin-hadoop3.tgz
 
 ```
 ./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspace-name>.<table-name>" /
+--conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
 --master "local[*]" /
 --class com.datastax.cdm.job.Migrate cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
@@ -44,7 +44,7 @@ Note:
 - Add option `--driver-memory 25G --executor-memory 25G` as shown below if the table migrated is large (over 100GB)
 ```
 ./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspace-name>.<table-name>" /
+--conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
 --master "local[*]" --driver-memory 25G --executor-memory 25G /
 --class com.datastax.cdm.job.Migrate cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
@@ -55,7 +55,7 @@ Note:
 
 ```
 ./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspace-name>.<table-name>" /
+--conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
 --master "local[*]" /
 --class com.datastax.cdm.job.DiffData cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
@@ -83,7 +83,7 @@ Note:
 - The validation job will never delete records from target i.e. it only adds or updates data on target
 
 # Migrating or Validating specific partition ranges
-- You can also use the tool to Migrate or Validate specific partition ranges by using a partition-file with the name `./<keyspace>.<tablename>_partitions.csv` in the below format in the current folder as input
+- You can also use the tool to Migrate or Validate specific partition ranges by using a partition-file with the name `./<keyspacename>.<tablename>_partitions.csv` in the below format in the current folder as input
 ```
 -507900353496146534,-107285462027022883
 -506781526266485690,1506166634797362039
@@ -94,21 +94,21 @@ Each line above represents a partition-range (`min,max`). Alternatively, you can
 
 ```
 spark-submit --properties-file cdm.properties /
- --conf spark.cdm.schema.origin.keyspaceTable="<keyspace-name>.<table-name>" /
- --conf spark.tokenRange.partitionFile="/<path-to-file>.<csv-input-filename>" /
+ --conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
+ --conf spark.tokenRange.partitionFile="/<path-to-file>/<csv-input-filename>" /
  --master "local[*]" /
  --class com.datastax.cdm.job.<Migrate|DiffData> cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
 This mode is specifically useful to processes a subset of partition-ranges that may have failed during a previous run.
 
 > **Note:**
-> A file named `./<keyspace>.<tablename>_partitions.csv` is auto generated by the Migration & Validation jobs in the above format containing any failed partition ranges. No file is created if there are no failed partitions. You can use this file as an input to process any failed partition in a following run.
+> A file named `./<keyspacename>.<tablename>_partitions.csv` is auto generated by the Migration & Validation jobs in the above format containing any failed partition ranges. No file is created if there are no failed partitions. You can use this file as an input to process any failed partition in a following run.
 
 # Perform large-field Guardrail violation checks
 - The tool can be used to identify large fields from a table that may break you cluster guardrails (e.g. AstraDB has a 10MB limit for a single large field)  `--class com.datastax.cdm.job.GuardrailCheck` as shown below
 ```
 ./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspace-name>.<table-name>" /
+--conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
 --conf spark.cdm.feature.guardrail.colSizeInKB=10000 /
 --master "local[*]" /
 --class com.datastax.cdm.job.GuardrailCheck cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt

SIT/features/06_partition_range/cdm.txt

@@ -1,2 +1,4 @@
-migrateData com.datastax.cdm.job.Migrate migrate.properties
-validateData com.datastax.cdm.job.DiffData migrate.properties
+migrateDataDefault com.datastax.cdm.job.Migrate migrate.properties
+validateDataDefault com.datastax.cdm.job.DiffData migrate.properties
+migrateData com.datastax.cdm.job.Migrate migrate_with_partitionfile.properties
+validateData com.datastax.cdm.job.DiffData migrate_with_partitionfile.properties

SIT/features/06_partition_range/execute.sh

@@ -3,9 +3,11 @@
 workingDir="$1"
 cd "$workingDir"
 
+/local/cdm.sh -f cdm.txt -s migrateDataDefault -d "$workingDir"
 /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir"
 
 cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/other.breakData.out 2> $workingDir/other.breakData.err
 
+/local/cdm.sh -f cdm.txt -s validateDataDefault -d "$workingDir"
 /local/cdm.sh -f cdm.txt -s validateData -d "$workingDir"
 

SIT/features/06_partition_range/migrate.properties

@@ -8,5 +8,3 @@ spark.cdm.perfops.numParts                        1
 spark.cdm.autocorrect.missing                     true
 spark.cdm.autocorrect.mismatch                    true
 
-spark.tokenrange.partitionFile                    ./partitions.csv
-

SIT/features/06_partition_range/migrate_with_partitionfile.properties

@@ -0,0 +1,12 @@
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
+
+spark.cdm.schema.origin.keyspaceTable             origin.feature_partition_range
+spark.cdm.schema.target.keyspaceTable             target.feature_partition_range
+spark.cdm.perfops.numParts                        1
+
+spark.cdm.autocorrect.missing                     true
+spark.cdm.autocorrect.mismatch                    true
+
+spark.tokenrange.partitionFile                    ./partitions.csv
+

SIT/features/06_partition_range/origin.feature_partition_range_partitions.csv

@@ -0,0 +1,2 @@
+0,2000000000000000000
+8100000000000000000,8500000000000000000

SIT/features/07_migrate_rows/cdm.txt

@@ -1 +1,2 @@
-migrateData com.datastax.cdm.job.MigrateRowsFromFile migrate.properties
+migrateDataDefault com.datastax.cdm.job.MigrateRowsFromFile migrate.properties
+migrateData com.datastax.cdm.job.MigrateRowsFromFile migrate_with_pkrowsfile.properties

SIT/features/07_migrate_rows/execute.sh

@@ -3,6 +3,7 @@
 workingDir="$1"
 cd "$workingDir"
 
+/local/cdm.sh -f cdm.txt -s migrateDataDefault -d "$workingDir"
 /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir"
 
 

SIT/features/07_migrate_rows/migrate.properties

@@ -4,5 +4,3 @@ spark.cdm.connect.target.host                     cdm-sit-cass
 spark.cdm.schema.origin.keyspaceTable             origin.feature_migrate_rows
 spark.cdm.schema.target.keyspaceTable             target.feature_migrate_rows
 spark.cdm.perfops.numParts                        1
-
-spark.tokenrange.partitionFile                    ./primary_key_rows.csv

SIT/features/07_migrate_rows/migrate_with_pkrowsfile.properties

@@ -0,0 +1,8 @@
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
+
+spark.cdm.schema.origin.keyspaceTable             origin.feature_migrate_rows
+spark.cdm.schema.target.keyspaceTable             target.feature_migrate_rows
+spark.cdm.perfops.numParts                        1
+
+spark.tokenrange.partitionFile                    ./primary_key_rows.csv