Skip to content

Commit 9002d43

Browse files
committed
Renamed artifact & updated readme
1 parent afd792a commit 9002d43

File tree

2 files changed

+11
-31
lines changed

2 files changed

+11
-31
lines changed

README.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# astra-spark-migration-ranges
1+
# cassandra-data-migrator
22

33
Spark jobs in this repo can be used for data migration and data validation.
44

@@ -19,15 +19,15 @@ tar -xvzf <spark downloaded file name>
1919
1. sparkConf.properties file needs to be configured as applicable for the environment
2020
> A sample Spark conf file configuration can be [found here](./src/resources/sparkConf.properties)
2121
2. Place the conf file where it can be accessed while running the job via spark-submit.
22-
3. Generate a fat jar (`migrate-0.x.jar`) using command `mvn clean package`
22+
3. Generate a fat jar (`cassandra-data-migrator-1.x.jar`) using command `mvn clean package`
2323
4. Run the 'Data Migration' job using `spark-submit` command as shown below:
2424

2525
```
2626
./spark-submit --properties-file sparkConf.properties /
2727
--master "local[*]" /
2828
--conf spark.migrate.source.minPartition=-9223372036854775808 /
2929
--conf spark.migrate.source.maxPartition=9223372036854775807 /
30-
--class datastax.astra.migrate.Migrate migrate-0.x.jar &> logfile_name.txt
30+
--class datastax.astra.migrate.Migrate cassandra-data-migrator-1.x.jar &> logfile_name.txt
3131
```
3232

3333
Note: Above command also generates a log file `logfile_name.txt` to avoid log output on the console.
@@ -42,7 +42,7 @@ Note: Above command also generates a log file `logfile_name.txt` to avoid log ou
4242
--master "local[*]" /
4343
--conf spark.migrate.source.minPartition=-9223372036854775808 /
4444
--conf spark.migrate.source.maxPartition=9223372036854775807 /
45-
--class datastax.astra.migrate.DiffData migrate-0.x.jar &> logfile_name.txt
45+
--class datastax.astra.migrate.DiffData cassandra-data-migrator-1.x.jar &> logfile_name.txt
4646
```
4747

4848
- Validation job will report differences as “ERRORS” in the log file as shown below

pom.xml

Lines changed: 7 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,72 +2,52 @@
22
<modelVersion>4.0.0</modelVersion>
33

44
<groupId>datastax.astra.migrate</groupId>
5-
<artifactId>migrate</artifactId>
6-
<version>0.18</version>
5+
<artifactId>cassandra-data-migrator</artifactId>
6+
<version>1.0</version>
77
<packaging>jar</packaging>
88

99
<properties>
1010
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
1111
<scala.version>2.11.8</scala.version>
1212
<scala.main.version>2.11</scala.main.version>
13-
<spark.version>2.4.0</spark.version>
14-
<scalatest.version>3.0.0</scalatest.version>
15-
<connector.version>2.5.1</connector.version>
13+
<spark.version>2.4.8</spark.version>
14+
<scalatest.version>3.2.12</scalatest.version>
15+
<connector.version>2.5.2</connector.version>
1616
<cassandra.version>3.11.6</cassandra.version>
17-
<junit.version>4.12</junit.version>
17+
<junit.version>4.13.2</junit.version>
1818
</properties>
1919

20-
<!--
21-
Please make sure that following dependencies have versions corresponding to the ones in your cluster.
22-
Note that spark-cassandra-connector should be provided with '&#45;&#45;packages' flag to spark-submit command.
23-
-->
2420
<dependencies>
2521
<dependency>
2622
<groupId>com.google.guava</groupId>
2723
<artifactId>guava</artifactId>
28-
<version>18.0</version>
24+
<version>31.1-jre</version>
2925
</dependency>
3026
<dependency>
3127
<groupId>org.scala-lang</groupId>
3228
<artifactId>scala-library</artifactId>
3329
<version>${scala.version}</version>
34-
<!-- <scope>provided</scope>-->
3530
</dependency>
3631
<dependency>
3732
<groupId>org.apache.spark</groupId>
3833
<artifactId>spark-core_${scala.main.version}</artifactId>
3934
<version>${spark.version}</version>
40-
<!-- <scope>provided</scope>-->
4135
</dependency>
4236
<dependency>
4337
<groupId>org.apache.spark</groupId>
4438
<artifactId>spark-sql_${scala.main.version}</artifactId>
4539
<version>${spark.version}</version>
46-
<!-- <scope>provided</scope>-->
4740
</dependency>
4841
<dependency>
4942
<groupId>org.apache.spark</groupId>
5043
<artifactId>spark-hive_${scala.main.version}</artifactId>
5144
<version>${spark.version}</version>
52-
<!-- <scope>provided</scope>-->
5345
</dependency>
5446
<dependency>
5547
<groupId>com.datastax.spark</groupId>
5648
<artifactId>spark-cassandra-connector_${scala.main.version}</artifactId>
5749
<version>${connector.version}</version>
58-
<!-- <scope>provided</scope>-->
5950
</dependency>
60-
<!--Your dependencies, 'provided' are not included in jar-->
61-
<!--<dependency>-->
62-
<!--<groupId>org.apache.commons</groupId>-->
63-
<!--<artifactId>commons-math3</artifactId>-->
64-
<!--<version>3.6.1</version>-->
65-
<!--</dependency>-->
66-
<!--<dependency>-->
67-
<!--<groupId>org.apache.commons</groupId>-->
68-
<!--<artifactId>commons-csv</artifactId>-->
69-
<!--<version>1.0</version>-->
70-
<!--</dependency>-->
7151

7252
<!-- Test Dependencies -->
7353
<dependency>

0 commit comments

Comments
 (0)