Merge pull request #174 from datastax/feature/CDM-88

Feature/cdm 88

Author: pravinbhat

README.md

@@ -34,7 +34,7 @@ tar -xvzf spark-3.3.1-bin-hadoop3.tgz
 
 ```
 ./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspace-name>.<table-name>" /
+--conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
 --master "local[*]" /
 --class com.datastax.cdm.job.Migrate cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
@@ -44,7 +44,7 @@ Note:
 - Add option `--driver-memory 25G --executor-memory 25G` as shown below if the table migrated is large (over 100GB)
 ```
 ./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspace-name>.<table-name>" /
+--conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
 --master "local[*]" --driver-memory 25G --executor-memory 25G /
 --class com.datastax.cdm.job.Migrate cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
@@ -55,7 +55,7 @@ Note:
 
 ```
 ./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspace-name>.<table-name>" /
+--conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
 --master "local[*]" /
 --class com.datastax.cdm.job.DiffData cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
@@ -82,46 +82,33 @@ spark.cdm.autocorrect.mismatch                    false|true
 Note:
 - The validation job will never delete records from target i.e. it only adds or updates data on target
 
-# Migrating specific partition ranges
-- You can also use the tool to migrate specific partition ranges using class option `--class com.datastax.cdm.job.MigratePartitionsFromFile` as shown below
-```
-./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspace-name>.<table-name>" /
---master "local[*]" /
---class com.datastax.cdm.job.MigratePartitionsFromFile cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
-```
-
-When running in above mode the tool assumes a `partitions.csv` file to be present in the current folder in the below format, where each line (`min,max`) represents a partition-range
+# Migrating or Validating specific partition ranges
+- You can also use the tool to Migrate or Validate specific partition ranges by using a partition-file with the name `./<keyspacename>.<tablename>_partitions.csv` in the below format in the current folder as input
 ```
 -507900353496146534,-107285462027022883
 -506781526266485690,1506166634797362039
 2637884402540451982,4638499294009575633
 798869613692279889,8699484505161403540
 ```
-This mode is specifically useful to processes a subset of partition-ranges that may have failed during a previous run.
-
-> **Note:**
-> A file ending with `*_partitions.csv` will be auto created by the Migration & Validation job in the above format containing any failed partition ranges. Just rename it as below & run the above job.
+Each line above represents a partition-range (`min,max`). Alternatively, you can also pass the partition-file via command-line param as shown below
 
 ```
-mv <keyspace>.<table>_partitions.csv partitions.csv
-```
-# Data validation for specific partition ranges
-- You can also use the tool to validate data for a specific partition ranges using class option `--class com.datastax.cdm.job.DiffPartitionsFromFile` as shown below,
-```
-./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspace-name>.<table-name>" /
---master "local[*]" /
---class com.datastax.cdm.job.DiffPartitionsFromFile cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
+spark-submit --properties-file cdm.properties /
+ --conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
+ --conf spark.tokenRange.partitionFile="/<path-to-file>/<csv-input-filename>" /
+ --master "local[*]" /
+ --class com.datastax.cdm.job.<Migrate|DiffData> cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
+This mode is specifically useful to processes a subset of partition-ranges that may have failed during a previous run.
 
-When running in above mode the tool assumes a `partitions.csv` file to be present in the current folder.
+> **Note:**
+> A file named `./<keyspacename>.<tablename>_partitions.csv` is auto generated by the Migration & Validation jobs in the above format containing any failed partition ranges. No file is created if there are no failed partitions. You can use this file as an input to process any failed partition in a following run.
 
 # Perform large-field Guardrail violation checks
 - The tool can be used to identify large fields from a table that may break you cluster guardrails (e.g. AstraDB has a 10MB limit for a single large field)  `--class com.datastax.cdm.job.GuardrailCheck` as shown below
 ```
 ./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspace-name>.<table-name>" /
+--conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
 --conf spark.cdm.feature.guardrail.colSizeInKB=10000 /
 --master "local[*]" /
 --class com.datastax.cdm.job.GuardrailCheck cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
@@ -132,7 +119,7 @@ When running in above mode the tool assumes a `partitions.csv` file to be presen
     - Including counter table [Counter tables](https://docs.datastax.com/en/dse/6.8/cql/cql/cql_using/useCountersConcept.html)
 - Preserve [writetimes](https://docs.datastax.com/en/dse/6.8/cql/cql/cql_reference/cql_commands/cqlSelect.html#cqlSelect__retrieving-the-datetime-a-write-occurred-p) and [TTLs](https://docs.datastax.com/en/dse/6.8/cql/cql/cql_reference/cql_commands/cqlSelect.html#cqlSelect__ref-select-ttl-p)
 - Supports migration/validation of advanced DataTypes ([Sets](https://docs.datastax.com/en/dse/6.8/cql/cql/cql_reference/refDataTypes.html#refDataTypes__set), [Lists](https://docs.datastax.com/en/dse/6.8/cql/cql/cql_reference/refDataTypes.html#refDataTypes__list), [Maps](https://docs.datastax.com/en/dse/6.8/cql/cql/cql_reference/refDataTypes.html#refDataTypes__map), [UDTs](https://docs.datastax.com/en/dse/6.8/cql/cql/cql_reference/refDataTypes.html#refDataTypes__udt))
-- Filter records from `Origin` using `writetimes` and/or CQL conditions and/or min/max token-range
+- Filter records from `Origin` using `writetimes` and/or CQL conditions and/or a list of token-ranges
 - Perform guardrail checks (identify large fields)
 - Supports adding `constants` as new columns on `Target`
 - Supports expanding `Map` columns on `Origin` into multiple records on `Target`

RELEASE.md

@@ -1,4 +1,7 @@
 # Release Notes
+## [4.1.0] - 2023-06-20
+- Refactored exception handling and loading of token-range filters to use the same Migrate & DiffData jobs instead of separate jobs to reduce code & maintenance overhead
+
 ## [4.0.2] - 2023-06-16
 - Capture failed partitions in a file for easier reruns 
 - Optimized mvn to reduce jar size

SIT/features/06_partition_range/cdm.txt

@@ -1,2 +1,4 @@
-migrateData com.datastax.cdm.job.MigratePartitionsFromFile migrate.properties
-validateData com.datastax.cdm.job.DiffPartitionsFromFile migrate.properties
+migrateDataDefault com.datastax.cdm.job.Migrate migrate.properties
+validateDataDefault com.datastax.cdm.job.DiffData migrate.properties
+migrateData com.datastax.cdm.job.Migrate migrate_with_partitionfile.properties
+validateData com.datastax.cdm.job.DiffData migrate_with_partitionfile.properties

SIT/features/06_partition_range/execute.sh

@@ -3,9 +3,11 @@
 workingDir="$1"
 cd "$workingDir"
 
+/local/cdm.sh -f cdm.txt -s migrateDataDefault -d "$workingDir"
 /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir"
 
 cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/other.breakData.out 2> $workingDir/other.breakData.err
 
+/local/cdm.sh -f cdm.txt -s validateDataDefault -d "$workingDir"
 /local/cdm.sh -f cdm.txt -s validateData -d "$workingDir"
 

SIT/features/06_partition_range/migrate_with_partitionfile.properties

@@ -0,0 +1,12 @@
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
+
+spark.cdm.schema.origin.keyspaceTable             origin.feature_partition_range
+spark.cdm.schema.target.keyspaceTable             target.feature_partition_range
+spark.cdm.perfops.numParts                        1
+
+spark.cdm.autocorrect.missing                     true
+spark.cdm.autocorrect.mismatch                    true
+
+spark.tokenrange.partitionFile                    ./partitions.csv
+

SIT/features/06_partition_range/origin.feature_partition_range_partitions.csv

@@ -0,0 +1,2 @@
+0,2000000000000000000
+8100000000000000000,8500000000000000000

SIT/features/07_migrate_rows/cdm.txt

@@ -1 +1,2 @@
-migrateData com.datastax.cdm.job.MigrateRowsFromFile migrate.properties
+migrateDataDefault com.datastax.cdm.job.MigrateRowsFromFile migrate.properties
+migrateData com.datastax.cdm.job.MigrateRowsFromFile migrate_with_pkrowsfile.properties

SIT/features/07_migrate_rows/execute.sh

@@ -3,6 +3,7 @@
 workingDir="$1"
 cd "$workingDir"
 
+/local/cdm.sh -f cdm.txt -s migrateDataDefault -d "$workingDir"
 /local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir"
 
 

SIT/features/07_migrate_rows/migrate.properties

@@ -4,5 +4,3 @@ spark.cdm.connect.target.host                     cdm-sit-cass
 spark.cdm.schema.origin.keyspaceTable             origin.feature_migrate_rows
 spark.cdm.schema.target.keyspaceTable             target.feature_migrate_rows
 spark.cdm.perfops.numParts                        1
-
-

SIT/features/07_migrate_rows/migrate_with_pkrowsfile.properties

@@ -0,0 +1,8 @@
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
+
+spark.cdm.schema.origin.keyspaceTable             origin.feature_migrate_rows
+spark.cdm.schema.target.keyspaceTable             target.feature_migrate_rows
+spark.cdm.perfops.numParts                        1
+
+spark.tokenrange.partitionFile                    ./primary_key_rows.csv