Merge pull request #241 from datastax/ms/jdk17

Bump Spark to 3.4.2 and Java to 11

Author: mfmaher2

.github/workflows/cdm-integrationtest.yml

@@ -14,12 +14,12 @@ jobs:
   CDM-Integration-Test:
     strategy:
       matrix:
-        jdk: [ 8 ]
+        jdk: [ 11, 17, 21 ]
         os: [ ubuntu-latest ]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
-      - name: Set up JDK 8
+      - name: Set up JDK
         uses: actions/setup-java@v3
         with:
           java-version: ${{ matrix.jdk }}

.github/workflows/maven-publish.yml

@@ -18,10 +18,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up JDK 8
+    - name: Set up JDK
       uses: actions/setup-java@v3
       with:
-        java-version: '8'
+        java-version: '11'
         distribution: 'temurin'
         cache: maven
         server-id: github # Value of the distributionManagement/repository/id field of the pom.xml

.github/workflows/maven.yml

@@ -23,14 +23,14 @@ jobs:
   build:
     strategy:
       matrix:
-        jdk: [ 8 ]
+        jdk: [ 11, 17, 21 ]
         os: [ ubuntu-latest ]
 
     runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up JDK 8
+    - name: Set up JDK
       uses: actions/setup-java@v3
       with:
         java-version: ${{ matrix.jdk }}

Dockerfile

@@ -1,4 +1,4 @@
-FROM eclipse-temurin:8-jammy
+FROM eclipse-temurin:11-jammy
 
 # Download all migration dependencies
 RUN mkdir -p /assets/ && cd /assets && \
@@ -9,9 +9,9 @@ RUN mkdir -p /assets/ && cd /assets && \
     curl -OL https://downloads.datastax.com/enterprise/cqlsh-astra.tar.gz && \
     tar -xzf ./cqlsh-astra.tar.gz && \
     rm ./cqlsh-astra.tar.gz && \
-    curl -OL https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz && \
-    tar -xzf ./spark-3.4.1-bin-hadoop3-scala2.13.tgz && \
-    rm ./spark-3.4.1-bin-hadoop3-scala2.13.tgz
+    curl -OL https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3-scala2.13.tgz && \
+    tar -xzf ./spark-3.4.2-bin-hadoop3-scala2.13.tgz && \
+    rm ./spark-3.4.2-bin-hadoop3-scala2.13.tgz
 
 RUN apt-get update && apt-get install -y openssh-server vim python3 --no-install-recommends && \
     rm -rf /var/lib/apt/lists/* && \
@@ -46,7 +46,7 @@ RUN chmod +x ./get-latest-maven-version.sh && \
     rm -rf "$USER_HOME_DIR/.m2"
 
 # Add all migration tools to path
-ENV PATH="${PATH}:/assets/dsbulk/bin/:/assets/cqlsh-astra/bin/:/assets/spark-3.4.1-bin-hadoop3-scala2.13/bin/"
+ENV PATH="${PATH}:/assets/dsbulk/bin/:/assets/cqlsh-astra/bin/:/assets/spark-3.4.2-bin-hadoop3-scala2.13/bin/"
 
 EXPOSE 22
 

README.md

@@ -7,7 +7,7 @@
 
 Migrate and Validate Tables between Origin and Target Cassandra Clusters.
 
-> :warning: Please note this job has been tested with spark version [3.4.1](https://archive.apache.org/dist/spark/spark-3.4.1/)
+> :warning: Please note this job has been tested with spark version [3.4.2](https://archive.apache.org/dist/spark/spark-3.4.2/)
 
 ## Install as a Container
 - Get the latest image that includes all dependencies from [DockerHub](https://hub.docker.com/r/datastax/cassandra-data-migrator)
@@ -17,11 +17,11 @@ Migrate and Validate Tables between Origin and Target Cassandra Clusters.
 - Download the latest jar file from the GitHub [packages area here](https://github.com/datastax/cassandra-data-migrator/packages/1832128)
 
 ### Prerequisite
-- Install Java8 as spark binaries are compiled with it.
-- Install Spark version [3.4.1](https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz) on a single VM (no cluster necessary) where you want to run this job. Spark can be installed by running the following: -
+- Install **Java11** (minimum) as Spark binaries are compiled with it.
+- Install Spark version [`3.4.2`](https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3-scala2.13.tgz) on a single VM (no cluster necessary) where you want to run this job. Spark can be installed by running the following: -
 ```
-wget https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz
-tar -xvzf spark-3.4.1-bin-hadoop3-scala2.13.tgz
+wget https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3-scala2.13.tgz
+tar -xvzf spark-3.4.2-bin-hadoop3-scala2.13.tgz
 ```
 
 > :warning: If the above Spark and Scala version is not properly installed, you'll then see a similar exception like below when running the CDM jobs,

RELEASE.md

@@ -1,4 +1,8 @@
 # Release Notes
+## [4.1.12] - 2024-01-22
+- Upgraded to use Spark `3.4.2`.
+- Added Java `11` as the minimally required pre-requisite to run CDM jobs.
+
 ## [4.1.9 to 4.1.11] - 2023-12-11
 - Code test & coverage changes
 

SIT/environment.sh

@@ -69,7 +69,8 @@ fi
 # These variables are hard-coded for now
 SUBNET=$(echo ${CIDR} | cut -d. -f1-3)
 CASS_VERSION=4
-CDM_VERSION=latest
+#CDM_VERSION=latest
+CDM_VERSION=ms-jdk17
 #==============================================================================================================================
 # Helper Functions
 #==============================================================================================================================

pom.xml

@@ -17,6 +17,9 @@
     <mockito.version>4.11.0</mockito.version>
     <java-driver.version>4.17.0</java-driver.version>
     <log4j.version>2.21.0</log4j.version>
+    <maven.compiler.source>11</maven.compiler.source>
+    <maven.compiler.target>11</maven.compiler.target>
+    <maven.compiler.release>11</maven.compiler.release>
   </properties>
 
   <distributionManagement>
@@ -182,7 +185,7 @@
     <connection>scm:git:[email protected]:datastax/cassandra-data-migrator.git</connection>
     <developerConnection>scm:git:[email protected]:datastax/cassandra-data-migrator.git</developerConnection>
     <url>https://github.com/datastax/cassandra-data-migrator</url>
-    <tag>4.1.8</tag>
+    <tag>4.1.11</tag>
   </scm>
 
   <build>
@@ -246,10 +249,11 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.10.1</version>
+        <version>3.12.1</version>
         <configuration>
-          <source>1.8</source>
-          <target>1.8</target>
+          <source>${maven.compiler.source}</source>
+          <target>${maven.compiler.target}</target>
+          <release>${maven.compiler.release}</release>
         </configuration>
       </plugin>
 

src/resources/migrate_data.sh

@@ -35,7 +35,7 @@
 ###########################################################################################################################
 
 # Path to spark-submit
-SPARK_SUBMIT=/home/ubuntu/spark-3.4.1-bin-hadoop3-scala2.13/bin/spark-submit
+SPARK_SUBMIT=/home/ubuntu/spark-3.4.2-bin-hadoop3-scala2.13/bin/spark-submit
 
 # Path to spark configuration for the table
 PROPS_FILE=/home/ubuntu/sparkConf.properties