Merge pull request #44 from datastax/feature/cdm-docker-image

Containerize

Author: pravinbhat

Dockerfile

@@ -0,0 +1,35 @@
+FROM eclipse-temurin:8-jammy
+
+# Add all migration tools to path
+RUN mkdir -p /assets/
+
+# Download all migration dependencies
+RUN cd /assets && \
+    curl -OL https://downloads.datastax.com/dsbulk/dsbulk.tar.gz && \
+    tar -xzf ./dsbulk.tar.gz && \
+    rm ./dsbulk.tar.gz && \
+    mv /assets/dsbulk-1.10.0 /assets/dsbulk  && \
+    curl -OL https://downloads.datastax.com/enterprise/cqlsh-astra.tar.gz && \
+    tar -xzf ./cqlsh-astra.tar.gz && \
+    rm ./cqlsh-astra.tar.gz && \
+    curl -OL https://archive.apache.org/dist/spark/spark-2.4.8/spark-2.4.8-bin-hadoop2.7.tgz && \
+    tar -xzf ./spark-2.4.8-bin-hadoop2.7.tgz && \
+    rm ./spark-2.4.8-bin-hadoop2.7.tgz
+
+RUN apt-get update && apt-get install -y openssh-server vim --no-install-recommends && \
+     rm -rf /var/lib/apt/lists/*
+RUN service ssh start
+
+# Copy CDM jar & template files
+COPY ./target/cassandra-data-migrator-*.jar /assets/
+COPY ./src/resources/sparkConf.properties /assets/
+COPY ./src/resources/partitions.csv /assets/
+COPY ./src/resources/primary_key_rows.csv /assets/
+COPY ./src/resources/runCommands.txt /assets/
+
+# Add all migration tools to path
+ENV PATH="${PATH}:/assets/dsbulk/bin/:/assets/cqlsh-astra/bin/:/assets/spark-2.4.8-bin-hadoop2.7/bin/"
+
+EXPOSE 22
+
+CMD ["/usr/sbin/sshd","-D"]

pom.xml

@@ -3,7 +3,7 @@
 
   <groupId>datastax.astra.migrate</groupId>
   <artifactId>cassandra-data-migrator</artifactId>
-  <version>2.9</version>
+  <version>2.10</version>
   <packaging>jar</packaging>
 
   <properties>

src/resources/100PartitionSplits.txt

@@ -39,7 +39,7 @@ if [ $S_IDX -lt -9000000000000000000 ]
 then
   E_IDX=-9000000000000000001
   echo "Running Migrate for Partition Range $S_IDX to $E_IDX .."
-  $SPARK_SUBMIT --properties-file $PROPS_FILE --master "local[*]" --conf spark.source.minPartition=$S_IDX --conf spark.source.maxPartition=$E_IDX --class datastax.astra.migrate.Migrate cassandra-data-migrator-*.jar
+  $SPARK_SUBMIT --properties-file $PROPS_FILE --master "local[*]" --conf spark.origin.minPartition=$S_IDX --conf spark.origin.maxPartition=$E_IDX --class datastax.astra.migrate.Migrate cassandra-data-migrator-*.jar
   S_IDX=-9000000000000000000
 fi
 
@@ -53,12 +53,12 @@ do
     E_IDX=$(( $S_IDX + $SLICE ))
   fi
   echo "Running Migrate for Partition Range $S_IDX to $E_IDX .."
-  $SPARK_SUBMIT --properties-file $PROPS_FILE --master "local[*]" --conf spark.source.minPartition=$S_IDX --conf spark.source.maxPartition=$E_IDX --class datastax.astra.migrate.Migrate cassandra-data-migrator-*.jar
+  $SPARK_SUBMIT --properties-file $PROPS_FILE --master "local[*]" --conf spark.origin.minPartition=$S_IDX --conf spark.origin.maxPartition=$E_IDX --class datastax.astra.migrate.Migrate cassandra-data-migrator-*.jar
   S_IDX=$(( $E_IDX + 1 ))
 done
 
 # Migrate final partition tokens from 9000000000000000000 to max-long
 E_IDX=9223372036854775807
 echo "Running Migrate for Partition Range $S_IDX to 9223372036854775807 .."
-$SPARK_SUBMIT --properties-file $PROPS_FILE --master "local[*]" --conf spark.source.minPartition=$S_IDX --conf spark.source.maxPartition=$E_IDX --class datastax.astra.migrate.Migrate cassandra-data-migrator-*.jar
+$SPARK_SUBMIT --properties-file $PROPS_FILE --master "local[*]" --conf spark.origin.minPartition=$S_IDX --conf spark.origin.maxPartition=$E_IDX --class datastax.astra.migrate.Migrate cassandra-data-migrator-*.jar
 echo "Completed Migration using $PROPS_FILE !!"

src/resources/migrate_data.sh

@@ -1,14 +1,14 @@
-// Download spark
-wget https://archive.apache.org/dist/spark/spark-2.4.8/spark-2.4.8-bin-hadoop2.6.tgz
-
-// Increase driver memory
---driver-memory 8G
+// Download dependencies
+curl -OL https://downloads.datastax.com/dsbulk/dsbulk.tar.gz
+curl -OL https://downloads.datastax.com/enterprise/cqlsh-astra.tar.gz
+wget https://archive.apache.org/dist/spark/spark-2.4.8/spark-2.4.8-bin-hadoop2.7.tgz
 
 // Migrate
-spark-submit --properties-file /<path>/sparkConf.properties --verbose --master "local[8]" --class datastax.astra.migrate.Migrate /<path>/cassandra-data-migrator-1.x.jar
+spark-submit --properties-file /<path>/sparkConf.properties --verbose --master "local[8]" --class datastax.astra.migrate.Migrate /<path>/cassandra-data-migrator-2.x.jar
+spark-submit --properties-file /<path>/sparkConf.properties --master "local[8]" --driver-memory 25G --executor-memory 25G --class datastax.astra.migrate.Migrate /<path>/cassandra-data-migrator-2.x.jar &> table_out.log
 
 // Random Partitioner Run Command
-spark-submit --properties-file /<path>/sparkConf.properties --verbose --master "local[8]" --conf spark.source.minPartition=-1 --conf spark.source.maxPartition=170141183460469231731687303715884105728 --class datastax.astra.migrate.Migrate /<path>/cassandra-data-migrator-1.x.jar
+spark-submit --properties-file /<path>/sparkConf.properties --verbose --master "local[8]" --conf spark.origin.minPartition=-1 --conf spark.origin.maxPartition=170141183460469231731687303715884105728 --class datastax.astra.migrate.Migrate /<path>/cassandra-data-migrator-2.x.jar
 
-// Diff Data
-spark-submit --properties-file /<path>/sparkConf.properties --verbose --master "local[8]" --class datastax.astra.migrate.DiffData /<path>/cassandra-data-migrator-1.x.jar
+// Validate
+spark-submit --properties-file /<path>/sparkConf.properties --master "local[8]" --driver-memory 25G --executor-memory 25G --class datastax.astra.migrate.DiffData /<path>/cassandra-data-migrator-2.x.jar &> table_out.log