@@ -16,36 +16,36 @@ class AbstractJob extends App {
16
16
17
17
val sc = spark.sparkContext
18
18
19
- val sourceIsAstra = sc.getConf.get(" spark.migrate. source.isAstra" , " false" )
20
- val sourceScbPath = sc.getConf.get(" spark.migrate. source.scb" , " " )
21
- val sourceHost = sc.getConf.get(" spark.migrate. source.host" , " " )
22
- val sourceUsername = sc.getConf.get(" spark.migrate. source.username" )
23
- val sourcePassword = sc.getConf.get(" spark.migrate. source.password" )
24
- val sourceReadConsistencyLevel = sc.getConf.get(" spark.migrate. source.read.consistency.level" , " LOCAL_QUORUM" )
25
- val sourceTrustStorePath = sc.getConf.get(" spark.migrate. source.trustStore.path" , " " )
26
- val sourceTrustStorePassword = sc.getConf.get(" spark.migrate. source.trustStore.password" , " " )
27
- val sourceTrustStoreType = sc.getConf.get(" spark.migrate. source.trustStore.type" , " JKS" )
28
- val sourceKeyStorePath = sc.getConf.get(" spark.migrate. source.keyStore.path" , " " )
29
- val sourceKeyStorePassword = sc.getConf.get(" spark.migrate. source.keyStore.password" , " " )
30
- val sourceEnabledAlgorithms = sc.getConf.get(" spark.migrate. source.enabledAlgorithms" , " " )
31
-
32
- val destinationIsAstra = sc.getConf.get(" spark.migrate. destination.isAstra" , " true" )
33
- val destinationScbPath = sc.getConf.get(" spark.migrate. destination.scb" , " " )
34
- val destinationHost = sc.getConf.get(" spark.migrate. destination.host" , " " )
35
- val destinationUsername = sc.getConf.get(" spark.migrate. destination.username" )
36
- val destinationPassword = sc.getConf.get(" spark.migrate. destination.password" )
37
- val destinationReadConsistencyLevel = sc.getConf.get(" spark.migrate. destination.read.consistency.level" , " LOCAL_QUORUM" )
38
- val destinationTrustStorePath = sc.getConf.get(" spark.migrate. destination.trustStore.path" , " " )
39
- val destinationTrustStorePassword = sc.getConf.get(" spark.migrate. destination.trustStore.password" , " " )
40
- val destinationTrustStoreType = sc.getConf.get(" spark.migrate. destination.trustStore.type" , " JKS" )
41
- val destinationKeyStorePath = sc.getConf.get(" spark.migrate. destination.keyStore.path" , " " )
42
- val destinationKeyStorePassword = sc.getConf.get(" spark.migrate. destination.keyStore.password" , " " )
43
- val destinationEnabledAlgorithms = sc.getConf.get(" spark.migrate. destination.enabledAlgorithms" , " " )
44
-
45
- val minPartition = new BigInteger (sc.getConf.get(" spark.migrate. source.minPartition" ))
46
- val maxPartition = new BigInteger (sc.getConf.get(" spark.migrate. source.maxPartition" ))
47
- val coveragePercent = sc.getConf.get(" spark.migrate. coveragePercent" , " 100" )
48
- val splitSize = sc.getConf.get(" spark.migrate. splitSize" , " 10000" )
19
+ val sourceIsAstra = sc.getConf.get(" spark.source.isAstra" , " false" )
20
+ val sourceScbPath = sc.getConf.get(" spark.source.scb" , " " )
21
+ val sourceHost = sc.getConf.get(" spark.source.host" , " " )
22
+ val sourceUsername = sc.getConf.get(" spark.source.username" , " " )
23
+ val sourcePassword = sc.getConf.get(" spark.source.password" , " " )
24
+ val sourceReadConsistencyLevel = sc.getConf.get(" spark.source.read.consistency.level" , " LOCAL_QUORUM" )
25
+ val sourceTrustStorePath = sc.getConf.get(" spark.source.trustStore.path" , " " )
26
+ val sourceTrustStorePassword = sc.getConf.get(" spark.source.trustStore.password" , " " )
27
+ val sourceTrustStoreType = sc.getConf.get(" spark.source.trustStore.type" , " JKS" )
28
+ val sourceKeyStorePath = sc.getConf.get(" spark.source.keyStore.path" , " " )
29
+ val sourceKeyStorePassword = sc.getConf.get(" spark.source.keyStore.password" , " " )
30
+ val sourceEnabledAlgorithms = sc.getConf.get(" spark.source.enabledAlgorithms" , " " )
31
+
32
+ val destinationIsAstra = sc.getConf.get(" spark.destination.isAstra" , " true" )
33
+ val destinationScbPath = sc.getConf.get(" spark.destination.scb" , " " )
34
+ val destinationHost = sc.getConf.get(" spark.destination.host" , " " )
35
+ val destinationUsername = sc.getConf.get(" spark.destination.username" )
36
+ val destinationPassword = sc.getConf.get(" spark.destination.password" )
37
+ val destinationReadConsistencyLevel = sc.getConf.get(" spark.destination.read.consistency.level" , " LOCAL_QUORUM" )
38
+ val destinationTrustStorePath = sc.getConf.get(" spark.destination.trustStore.path" , " " )
39
+ val destinationTrustStorePassword = sc.getConf.get(" spark.destination.trustStore.password" , " " )
40
+ val destinationTrustStoreType = sc.getConf.get(" spark.destination.trustStore.type" , " JKS" )
41
+ val destinationKeyStorePath = sc.getConf.get(" spark.destination.keyStore.path" , " " )
42
+ val destinationKeyStorePassword = sc.getConf.get(" spark.destination.keyStore.password" , " " )
43
+ val destinationEnabledAlgorithms = sc.getConf.get(" spark.destination.enabledAlgorithms" , " " )
44
+
45
+ val minPartition = new BigInteger (sc.getConf.get(" spark.source.minPartition" , " -9223372036854775808 " ))
46
+ val maxPartition = new BigInteger (sc.getConf.get(" spark.source.maxPartition" , " 9223372036854775807 " ))
47
+ val coveragePercent = sc.getConf.get(" spark.coveragePercent" , " 100" )
48
+ val splitSize = sc.getConf.get(" spark.splitSize" , " 10000" )
49
49
val partitions = SplitPartitions .getRandomSubPartitions(BigInteger .valueOf(Long .parseLong(splitSize)), minPartition, maxPartition,Integer .parseInt(coveragePercent))
50
50
51
51
var sourceConnection = getConnection(true , sourceIsAstra, sourceScbPath, sourceHost, sourceUsername, sourcePassword, sourceReadConsistencyLevel,
@@ -68,15 +68,15 @@ class AbstractJob extends App {
68
68
}
69
69
70
70
if (" true" .equals(isAstra)) {
71
- abstractLogger.info(connType + " : Connected to Astra! " );
71
+ abstractLogger.info(connType + " : Connected to Astra using SCB: " + scbPath );
72
72
73
73
return CassandraConnector (sc.getConf
74
74
.set(" spark.cassandra.auth.username" , username)
75
75
.set(" spark.cassandra.auth.password" , password)
76
76
.set(" spark.cassandra.input.consistency.level" , readConsistencyLevel)
77
77
.set(" spark.cassandra.connection.config.cloud.path" , scbPath))
78
78
} else if (null != trustStorePath && ! trustStorePath.trim.isEmpty) {
79
- abstractLogger.info(connType + " : Connected to Cassandra (or DSE) with SSL! " );
79
+ abstractLogger.info(connType + " : Connected to Cassandra (or DSE) with SSL host: " + host );
80
80
81
81
// Use defaults when not provided
82
82
var enabledAlgorithmsVar = enabledAlgorithms
@@ -99,7 +99,7 @@ class AbstractJob extends App {
99
99
.set(" spark.cassandra.connection.ssl.clientAuth.enabled" , " true" )
100
100
)
101
101
} else {
102
- abstractLogger.info(connType + " : Connected to Cassandra (or DSE)! " );
102
+ abstractLogger.info(connType + " : Connected to Cassandra (or DSE) host: " + host );
103
103
104
104
return CassandraConnector (sc.getConf.set(" spark.cassandra.auth.username" , username)
105
105
.set(" spark.cassandra.auth.password" , password)
0 commit comments