Merge branch 'main' into feature/exception_handler_refactor

faizalrub-datastax · faizalrub-datastax · commit 9f38c0abe484 · 2023-06-07T17:49:42.000-04:00
# Conflicts:
#	src/main/java/com/datastax/cdm/job/AbstractJobSession.java
#	src/resources/cdm.properties
diff --git a/PERF/cdm-v4.properties b/PERF/cdm-v4.properties
@@ -20,8 +20,8 @@
 # Connections are made with the Spark Cassandra Connector; conventions there likely work here
 # See https://github.com/datastax/spark-cassandra-connector/blob/master/doc/reference.md
 #
-#  spark.cdm.origin.connect
-#  spark.cdm.target.connect
+#  spark.cdm.connect.origin
+#  spark.cdm.connect.target
 #   .host      : Default is localhost. Hostname/IP address of the cluster; may be a comma-separated list,  
 #                and can follow the <hostname>:<port> convention.
 #   .port      : Default is 9042. Port number to use if not specified on .host 
@@ -31,17 +31,17 @@
 #
 # You must set either .host or .scb.
 #-----------------------------------------------------------------------------------------------------------
-spark.cdm.origin.connect.host          cass-origin
-spark.cdm.origin.connect.port          9042
-#spark.cdm.origin.connect.scb           file:///aaa/bbb/secure-connect-enterprise.zip
-spark.cdm.origin.connect.username      cassandra
-spark.cdm.origin.connect.password      cassandra
+spark.cdm.connect.origin.host          cass-origin
+spark.cdm.connect.origin.port          9042
+#spark.cdm.connect.origin.scb           file:///aaa/bbb/secure-connect-enterprise.zip
+spark.cdm.connect.origin.username      cassandra
+spark.cdm.connect.origin.password      cassandra
 
-spark.cdm.target.connect.host          cass-target
-spark.cdm.target.connect.port          9042
-#spark.cdm.target.connect.scb           file:///aaa/bbb/secure-connect-enterprise.zip
-spark.cdm.target.connect.username      cassandra
-spark.cdm.target.connect.password      cassandra
+spark.cdm.connect.target.host          cass-target
+spark.cdm.connect.target.port          9042
+#spark.cdm.connect.target.scb           file:///aaa/bbb/secure-connect-enterprise.zip
+spark.cdm.connect.target.username      cassandra
+spark.cdm.connect.target.password      cassandra
 
 #===========================================================================================================
 # Details about the Origin Schema
@@ -326,8 +326,8 @@ spark.cdm.perfops.writeRateLimit                  5000
 # TLS (SSL) connection parameters, if so configured. Note that Secure Bundles embed these details.
 # See https://github.com/datastax/spark-cassandra-connector/blob/master/doc/reference.md
 #
-#  spark.cdm.origin.connect.tls
-#  spark.cdm.target.connect.tls
+#  spark.cdm.connect.origin.tls
+#  spark.cdm.connect.target.tls
 #    .enabled             : Default is false. Set to true if TLS is used.
 #    .trustStore          
 #       .path             : Filepath to the Java truststore file
@@ -338,20 +338,20 @@ spark.cdm.perfops.writeRateLimit                  5000
 #       .password         : Password needed to open the keystore
 #    .enabledAlgorithms   : Default is TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA
 #-----------------------------------------------------------------------------------------------------------
-#spark.cdm.origin.connect.tls.enabled              false
-#spark.cdm.origin.connect.tls.trustStore.path
-#spark.cdm.origin.connect.tls.trustStore.password
-#spark.cdm.origin.connect.tls.trustStore.type      JKS
-#spark.cdm.origin.connect.tls.keyStore.path
-#spark.cdm.origin.connect.tls.keyStore.password
-#spark.cdm.origin.connect.tls.enabledAlgorithms    TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA
+#spark.cdm.connect.origin.tls.enabled              false
+#spark.cdm.connect.origin.tls.trustStore.path
+#spark.cdm.connect.origin.tls.trustStore.password
+#spark.cdm.connect.origin.tls.trustStore.type      JKS
+#spark.cdm.connect.origin.tls.keyStore.path
+#spark.cdm.connect.origin.tls.keyStore.password
+#spark.cdm.connect.origin.tls.enabledAlgorithms    TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA
 
-#spark.cdm.target.connect.tls.enabled              false
-#spark.cdm.target.connect.tls.trustStore.path
-#spark.cdm.target.connect.tls.trustStore.password
-#spark.cdm.target.connect.tls.trustStore.type      JKS
-#spark.cdm.target.connect.tls.keyStore.path
-#spark.cdm.target.connect.tls.keyStore.password
-#spark.cdm.target.connect.tls.enabledAlgorithms    TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA
+#spark.cdm.connect.target.tls.enabled              false
+#spark.cdm.connect.target.tls.trustStore.path
+#spark.cdm.connect.target.tls.trustStore.password
+#spark.cdm.connect.target.tls.trustStore.type      JKS
+#spark.cdm.connect.target.tls.keyStore.path
+#spark.cdm.connect.target.tls.keyStore.password
+#spark.cdm.connect.target.tls.enabledAlgorithms    TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA
 
 
diff --git a/README.md b/README.md
@@ -26,8 +26,9 @@ tar -xvzf spark-3.3.1-bin-hadoop3.tgz
 
 > :warning: Note that Version 4 of the tool is not backward-compatible with .properties files created in previous versions, and that package names have changed.
 
-1. `cdm.properties` file needs to be configured as applicable for the environment. Parameter descriptions and defaults are described in the file.
-   > A sample properties file configuration can be [found here](./src/resources/cdm.properties)
+1. `cdm.properties` file needs to be configured as applicable for the environment. Parameter descriptions and defaults are described in the file. The file can have any name, it does not need to be `cdm.properties`.
+   > * A simplified sample properties file configuration can be found here as [cdm.properties](./src/resources/cdm.properties)
+   > * A complete sample properties file configuration can be found here as [cdm-detailed.properties](./src/resources/cdm-detailed.properties)
 2. Place the properties file where it can be accessed while running the job via spark-submit.
 3. Run the below job using `spark-submit` command as shown below:
 
@@ -119,12 +120,12 @@ When running in above mode the tool assumes a `partitions.csv` file to be presen
 # Perform large-field Guardrail violation checks
 - The tool can be used to identify large fields from a table that may break you cluster guardrails (e.g. AstraDB has a 10MB limit for a single large field)  `--class datastax.astra.migrate.Guardrail` as shown below
 ```
-./spark-submit --properties-file cdmGuardrail.properties /
+./spark-submit --properties-file cdm.properties /
 --conf spark.origin.keyspaceTable="<keyspace-name>.<table-name>" /
+--conf spark.cdm.feature.guardrail.colSizeInKB=10000 /
 --master "local[*]" /
 --class datastax.cdm.job.GuardrailCheck cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
-> A sample Guardrail properties file can be [found here](./src/resources/cdmGuardrail.properties)
 
 # Features
 - Auto-detects table schema (column names, types, keys, collections, UDTs, etc.)
diff --git a/SIT/features/01_constant_column/fix.properties b/SIT/features/01_constant_column/fix.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.feature_constant_column
 spark.cdm.schema.target.keyspaceTable             target.feature_constant_column
diff --git a/SIT/features/01_constant_column/migrate.properties b/SIT/features/01_constant_column/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.feature_constant_column
 spark.cdm.schema.target.keyspaceTable             target.feature_constant_column
diff --git a/SIT/features/02_explode_map/fix.properties b/SIT/features/02_explode_map/fix.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.feature_explode_map
 spark.cdm.schema.target.keyspaceTable             target.feature_explode_map
diff --git a/SIT/features/02_explode_map/migrate.properties b/SIT/features/02_explode_map/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.feature_explode_map
 spark.cdm.schema.target.keyspaceTable             target.feature_explode_map
diff --git a/SIT/features/03_codec/fix.properties b/SIT/features/03_codec/fix.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.codec
 spark.cdm.schema.target.keyspaceTable             target.codec
diff --git a/SIT/features/03_codec/migrate.properties b/SIT/features/03_codec/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.codec
 spark.cdm.schema.target.keyspaceTable             target.codec
diff --git a/SIT/features/04_udt_mapper/fix.properties b/SIT/features/04_udt_mapper/fix.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.udt_mapper
 spark.cdm.schema.target.keyspaceTable             target.udt_mapper
diff --git a/SIT/features/04_udt_mapper/migrate.properties b/SIT/features/04_udt_mapper/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.udt_mapper
 spark.cdm.schema.target.keyspaceTable             target.udt_mapper
diff --git a/SIT/features/05_guardrail/fix.properties b/SIT/features/05_guardrail/fix.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.feature_guardrail
 spark.cdm.schema.target.keyspaceTable             target.feature_guardrail
diff --git a/SIT/features/05_guardrail/migrate.properties b/SIT/features/05_guardrail/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.feature_guardrail
 spark.cdm.schema.target.keyspaceTable             target.feature_guardrail
diff --git a/SIT/features/06_partition_range/migrate.properties b/SIT/features/06_partition_range/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.feature_partition_range
 spark.cdm.schema.target.keyspaceTable             target.feature_partition_range
diff --git a/SIT/features/07_migrate_rows/migrate.properties b/SIT/features/07_migrate_rows/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.feature_migrate_rows
 spark.cdm.schema.target.keyspaceTable             target.feature_migrate_rows
diff --git a/SIT/regression/01_explode_map_with_constants/fix.properties b/SIT/regression/01_explode_map_with_constants/fix.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.feature_explode_map_with_constants
 spark.cdm.schema.target.keyspaceTable             target.feature_explode_map_with_constants
diff --git a/SIT/regression/01_explode_map_with_constants/migrate.properties b/SIT/regression/01_explode_map_with_constants/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.feature_explode_map_with_constants
 spark.cdm.schema.target.keyspaceTable             target.feature_explode_map_with_constants
diff --git a/SIT/regression/02_ColumnRenameWithConstantsAndExplode/fix.properties b/SIT/regression/02_ColumnRenameWithConstantsAndExplode/fix.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin."ColumnRenameWithConstantsAndExplode"
 spark.cdm.schema.target.keyspaceTable             target.column_rename_with_constants_and_explode
diff --git a/SIT/regression/02_ColumnRenameWithConstantsAndExplode/migrate.properties b/SIT/regression/02_ColumnRenameWithConstantsAndExplode/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin."ColumnRenameWithConstantsAndExplode"
 spark.cdm.schema.target.keyspaceTable             target.column_rename_with_constants_and_explode
diff --git a/SIT/regression/03_performance/fix.properties b/SIT/regression/03_performance/fix.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.regression_performance
 spark.cdm.schema.target.keyspaceTable             target.regression_performance
diff --git a/SIT/regression/03_performance/migrate.properties b/SIT/regression/03_performance/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.regression_performance
 spark.cdm.schema.target.keyspaceTable             target.regression_performance
diff --git a/SIT/regression/04_null_ts_in_pk/migrate.properties b/SIT/regression/04_null_ts_in_pk/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.regression_null_ts_in_pk
 spark.cdm.schema.target.keyspaceTable             target.regression_null_ts_in_pk
diff --git a/SIT/smoke/01_basic_kvp/migrate.properties b/SIT/smoke/01_basic_kvp/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.smoke_basic_kvp
 spark.cdm.schema.target.keyspaceTable             target.smoke_basic_kvp
diff --git a/SIT/smoke/02_autocorrect_kvp/migrate.properties b/SIT/smoke/02_autocorrect_kvp/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.smoke_autocorrect_kvp
 spark.cdm.schema.target.keyspaceTable             target.smoke_autocorrect_kvp
diff --git a/SIT/smoke/03_ttl_writetime/fix.properties b/SIT/smoke/03_ttl_writetime/fix.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.smoke_ttl_writetime
 spark.cdm.schema.target.keyspaceTable             target.smoke_ttl_writetime
diff --git a/SIT/smoke/03_ttl_writetime/migrate.properties b/SIT/smoke/03_ttl_writetime/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.smoke_ttl_writetime
 spark.cdm.schema.target.keyspaceTable             target.smoke_ttl_writetime
diff --git a/SIT/smoke/04_counters/fix-force.properties b/SIT/smoke/04_counters/fix-force.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.smoke_counters
 spark.cdm.schema.target.keyspaceTable             target.smoke_counters
diff --git a/SIT/smoke/04_counters/fix.properties b/SIT/smoke/04_counters/fix.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.smoke_counters
 spark.cdm.schema.target.keyspaceTable             target.smoke_counters
diff --git a/SIT/smoke/04_counters/migrate.properties b/SIT/smoke/04_counters/migrate.properties
@@ -1,5 +1,5 @@
-spark.cdm.origin.connect.host                     cdm-sit-cass
-spark.cdm.target.connect.host                     cdm-sit-cass
+spark.cdm.connect.origin.host                     cdm-sit-cass
+spark.cdm.connect.target.host                     cdm-sit-cass
 
 spark.cdm.schema.origin.keyspaceTable             origin.smoke_counters
 spark.cdm.schema.target.keyspaceTable             target.smoke_counters
diff --git a/src/main/java/com/datastax/cdm/job/AbstractJobSession.java b/src/main/java/com/datastax/cdm/job/AbstractJobSession.java
@@ -42,18 +42,18 @@ protected AbstractJobSession(CqlSession originSession, CqlSession targetSession,
             printStatsAfter = propertyHelper.getInteger(KnownProperties.PRINT_STATS_AFTER);
         }
 
-        readLimiter = RateLimiter.create(propertyHelper.getInteger(KnownProperties.PERF_LIMIT_READ));
-        writeLimiter = RateLimiter.create(propertyHelper.getInteger(KnownProperties.PERF_LIMIT_WRITE));
+        rateLimiterOrigin = RateLimiter.create(propertyHelper.getInteger(KnownProperties.PERF_RATELIMIT_ORIGIN));
+        rateLimiterTarget = RateLimiter.create(propertyHelper.getInteger(KnownProperties.PERF_RATELIMIT_TARGET));
         maxRetries = propertyHelper.getInteger(KnownProperties.MAX_RETRIES);
 
         tokenRangeExceptionDir = propertyHelper.getString(KnownProperties.TOKEN_RANGE_EXCEPTION_DIR);
         exceptionFileName = propertyHelper.getString(KnownProperties.ORIGIN_KEYSPACE_TABLE);
 
         logger.info("PARAM -- Max Retries: {}", maxRetries);
-        logger.info("PARAM -- ReadRateLimit: {}", readLimiter.getRate());
-        logger.info("PARAM -- WriteRateLimit: {}", writeLimiter.getRate());
         logger.info("PARAM -- Token range exception dir: {}", tokenRangeExceptionDir);
         logger.info("PARAM -- Token range exception file name: {}", exceptionFileName);
+        logger.info("PARAM -- Origin Rate Limit: {}", rateLimiterOrigin.getRate());
+        logger.info("PARAM -- Target Rate Limit: {}", rateLimiterTarget.getRate());
 
         this.originSession = new EnhancedSession(propertyHelper, originSession, true);
         this.targetSession = new EnhancedSession(propertyHelper, targetSession, false);
diff --git a/src/main/java/com/datastax/cdm/job/BaseJobSession.java b/src/main/java/com/datastax/cdm/job/BaseJobSession.java
@@ -24,8 +24,8 @@ public abstract class BaseJobSession {
     // then do the following to set the values as they are only applicable per JVM
     // (hence spark Executor)...
     // Rate = Total Throughput (write/read per sec) / Total Executors
-    protected RateLimiter readLimiter;
-    protected RateLimiter writeLimiter;
+    protected RateLimiter rateLimiterOrigin;
+    protected RateLimiter rateLimiterTarget;
     protected Integer maxRetries = 10;
 
     protected Integer printStatsAfter = 100000;
diff --git a/src/main/java/com/datastax/cdm/job/CopyJobSession.java b/src/main/java/com/datastax/cdm/job/CopyJobSession.java
diff --git a/src/main/java/com/datastax/cdm/job/CopyPKJobSession.java b/src/main/java/com/datastax/cdm/job/CopyPKJobSession.java
diff --git a/src/main/java/com/datastax/cdm/job/DiffJobSession.java b/src/main/java/com/datastax/cdm/job/DiffJobSession.java
diff --git a/src/main/java/com/datastax/cdm/job/GuardrailCheckJobSession.java b/src/main/java/com/datastax/cdm/job/GuardrailCheckJobSession.java
diff --git a/src/main/java/com/datastax/cdm/properties/KnownProperties.java b/src/main/java/com/datastax/cdm/properties/KnownProperties.java
diff --git a/src/main/java/com/datastax/cdm/properties/PropertyHelper.java b/src/main/java/com/datastax/cdm/properties/PropertyHelper.java
diff --git a/src/main/scala/com/datastax/cdm/job/ConnectionFetcher.scala b/src/main/scala/com/datastax/cdm/job/ConnectionFetcher.scala
diff --git a/src/resources/cdm-detailed.properties b/src/resources/cdm-detailed.properties
diff --git a/src/resources/cdm.properties b/src/resources/cdm.properties
diff --git a/src/test/java/com/datastax/cdm/properties/PropertyHelperTest.java b/src/test/java/com/datastax/cdm/properties/PropertyHelperTest.java
