Upgrade Spark and SCC versions to fix CVEs (#205)

* Upgrade all cylinders - Spark & SCC upgrade to 3.4.1, C* used in testing to 4 latest, and log4j versions

Author: msmygit

.github/workflows/maven-publish.yml

@@ -23,6 +23,7 @@ jobs:
       with:
         java-version: '8'
         distribution: 'temurin'
+        cache: maven
         server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
         settings-path: ${{ github.workspace }} # location for the settings.xml file
 

.gitignore

@@ -10,3 +10,5 @@ SIT/local
 .classpath
 .project
 .settings/*
+src/main/main.iml
+src/test/test.iml

Dockerfile

@@ -9,9 +9,9 @@ RUN mkdir -p /assets/ && cd /assets && \
     curl -OL https://downloads.datastax.com/enterprise/cqlsh-astra.tar.gz && \
     tar -xzf ./cqlsh-astra.tar.gz && \
     rm ./cqlsh-astra.tar.gz && \
-    curl -OL https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz && \
-    tar -xzf ./spark-3.3.1-bin-hadoop3.tgz && \
-    rm ./spark-3.3.1-bin-hadoop3.tgz
+    curl -OL https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz && \
+    tar -xzf ./spark-3.4.1-bin-hadoop3.tgz && \
+    rm ./spark-3.4.1-bin-hadoop3.tgz
 
 RUN apt-get update && apt-get install -y openssh-server vim python3 --no-install-recommends && \
     rm -rf /var/lib/apt/lists/*  && \
@@ -46,7 +46,7 @@ RUN chmod +x ./get-latest-maven-version.sh && \
     rm -rf "$USER_HOME_DIR/.m2"
 
 # Add all migration tools to path
-ENV PATH="${PATH}:/assets/dsbulk/bin/:/assets/cqlsh-astra/bin/:/assets/spark-3.3.1-bin-hadoop3/bin/"
+ENV PATH="${PATH}:/assets/dsbulk/bin/:/assets/cqlsh-astra/bin/:/assets/spark-3.4.1-bin-hadoop3/bin/"
 
 EXPOSE 22
 

README.md

@@ -7,7 +7,7 @@
 
 Migrate and Validate Tables between Origin and Target Cassandra Clusters.
 
-> :warning: Please note this job has been tested with spark version [3.3.1](https://archive.apache.org/dist/spark/spark-3.3.1/)
+> :warning: Please note this job has been tested with spark version [3.4.1](https://archive.apache.org/dist/spark/spark-3.4.1/)
 
 ## Install as a Container
 - Get the latest image that includes all dependencies from [DockerHub](https://hub.docker.com/r/datastax/cassandra-data-migrator)
@@ -18,10 +18,10 @@ Migrate and Validate Tables between Origin and Target Cassandra Clusters.
 
 ### Prerequisite
 - Install Java8 as spark binaries are compiled with it.
-- Install Spark version [3.3.1](https://archive.apache.org/dist/spark/spark-3.3.1/) on a single VM (no cluster necessary) where you want to run this job. Spark can be installed by running the following: -
+- Install Spark version [3.4.1](https://archive.apache.org/dist/spark/spark-3.4.1/) on a single VM (no cluster necessary) where you want to run this job. Spark can be installed by running the following: -
 ```
-wget https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz
-tar -xvzf spark-3.3.1-bin-hadoop3.tgz
+wget https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz
+tar -xvzf spark-3.4.1-bin-hadoop3.tgz
 ```
 
 # Steps for Data-Migration:

SIT/environment.sh

@@ -68,13 +68,13 @@ fi
 ###
 # These variables are hard-coded for now
 SUBNET=$(echo ${CIDR} | cut -d. -f1-3)
-CASS_VERSION=3
+CASS_VERSION=4
 CDM_VERSION=latest
 
 #==============================================================================================================================
 # Helper Functions
 #==============================================================================================================================
-# Common enviornment and functions
+# Common environment and functions
 . common.sh
 
 _testDockerNetwork() {

SIT/smoke/03_ttl_writetime/expected.out

@@ -6,7 +6,7 @@ record5|A---|B---|CCCC|DDDD|1087384200000000|6
 record2|AAAA|BBBB|CCCC|DDDD|1087383720000000|5
 record6|AAAA|BBBB|C---|DDDD|1087384200000000|5
 record1|AAAA|BBBB|CCCC|DDDD|1087383720000000|5
-record8|AAAA|BBBB|CCCC|DDDD|1087383720000000|6
+record8|A---|BBBB|CCCC|DDDD|1087383720000000|6
 record7|AAAA|B---|CCCC|DDDD|1087384200000000|6
 record4|AAAA|BBBB|CCCC|DDDD|1087384200000000|3
 

SIT/test.sh

@@ -31,7 +31,7 @@ if [[ ! -d ${PHASE} || $(ls -d ${PHASE}/* | wc -l) -eq 0 ]]; then
   _fatal "Phase directory ${PHASE} does not exist, or is empty"
 fi
 
-# Common enviornment and functions
+# Common environment and functions
 . common.sh
 
 _captureOutput() {

pom.xml

@@ -10,13 +10,14 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <scala.version>2.12.17</scala.version>
     <scala.main.version>2.12</scala.main.version>
-    <spark.version>3.3.1</spark.version>
+    <spark.version>3.4.1</spark.version>
     <scalatest.version>3.2.12</scalatest.version>
-    <connector.version>3.2.0</connector.version>
+    <connector.version>3.4.1</connector.version>
     <cassandra.version>5.0-alpha1</cassandra.version>
     <junit.version>5.9.1</junit.version>
     <mockito.version>4.11.0</mockito.version>
     <java-driver.version>4.17.0</java-driver.version>
+    <log4j.version>2.20.0</log4j.version>
   </properties>
 
   <distributionManagement>
@@ -46,6 +47,10 @@
           <groupId>org.apache.logging.log4j</groupId>
           <artifactId>log4j-slf4j-impl</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-slf4j2-impl</artifactId>
+        </exclusion>
       </exclusions>
       <scope>provided</scope>
     </dependency>
@@ -120,23 +125,23 @@
     <dependency>
       <groupId>com.github.jnr</groupId>
       <artifactId>jnr-posix</artifactId>
-      <version>3.1.15</version>
+      <version>3.1.18</version>
     </dependency>
 
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-api</artifactId>
-      <version>2.19.0</version>
+      <version>${log4j.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-core</artifactId>
-      <version>2.19.0</version>
+      <version>${log4j.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-to-slf4j</artifactId>
-      <version>2.19.0</version>
+      <version>${log4j.version}</version>
     </dependency>
 
     <!-- Test Dependencies -->
@@ -206,7 +211,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.4.1</version>
+        <version>3.5.1</version>
         <executions>
           <execution>