Merge branch 'main' into snyk-upgrade-dafe6fd4d1eb795d6504c32641f4b46f

Author: msmygit

.all-contributorsrc

@@ -139,6 +139,16 @@
       "contributions": [
         "review"
       ]
+    },
+    {
+      "login": "Jeremya",
+      "name": "Jeremy",
+      "avatar_url": "https://avatars.githubusercontent.com/u/576519?v=4",
+      "profile": "https://github.com/Jeremya",
+      "contributions": [
+        "code"
+      ]
     }
-  ]
+  ],
+  "commitType": "docs"
 }

.github/workflows/cdm-integrationtest.yml

@@ -1,14 +1,28 @@
 name: Build and test jar with integration tests
-on: [push]
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
+
 jobs:
   CDM-Integration-Test:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        jdk: [ 8 ]
+        os: [ ubuntu-latest ]
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK 8
         uses: actions/setup-java@v3
         with:
-          java-version: '8'
+          java-version: ${{ matrix.jdk }}
           distribution: 'temurin'
           cache: maven
       - name: Test SIT with cdm

.github/workflows/maven.yml

@@ -8,19 +8,32 @@
 
 name: Java CI with Maven
 
-on: [push, pull_request]
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
 
 jobs:
   build:
+    strategy:
+      matrix:
+        jdk: [ 8 ]
+        os: [ ubuntu-latest ]
 
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v3
     - name: Set up JDK 8
       uses: actions/setup-java@v3
       with:
-        java-version: '8'
+        java-version: ${{ matrix.jdk }}
         distribution: 'temurin'
         cache: maven
     - name: Build with Maven

.github/workflows/snyk-cli-scan.yml

@@ -10,5 +10,11 @@ on:
     branches: [ main ]
   workflow_dispatch:
 
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  #group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job || github.run_id }}
+  cancel-in-progress: true
+
 env:
   SNYK_SEVERITY_THRESHOLD_LEVEL: critical
+

.github/workflows/snyk-pr-cleanup.yml

@@ -9,3 +9,8 @@ on:
       - main
   workflow_dispatch:
 
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  #group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job || github.run_id }}
+  cancel-in-progress: true
+

.settings/org.eclipse.core.resources.prefs

@@ -149,6 +149,7 @@ For recognizing contributions, please follow [this documentation](https://allcon
       <td align="center" valign="top" width="16.66%"><a href="https://github.com/vaishakhbn"><img src="https://avatars.githubusercontent.com/u/2619002?v=4?s=50" width="50px;" alt="Vaishakh Baragur Narasimhareddy"/><br /><sub><b>Vaishakh Baragur Narasimhareddy</b></sub></a><br /><a href="https://github.com/datastax/cassandra-data-migrator/commits?author=vaishakhbn" title="Code">💻</a> <a href="https://github.com/datastax/cassandra-data-migrator/commits?author=vaishakhbn" title="Tests">⚠️</a></td>
       <td align="center" valign="top" width="16.66%"><a href="https://github.com/mieslep"><img src="https://avatars.githubusercontent.com/u/5420540?v=4?s=50" width="50px;" alt="Phil Miesle"/><br /><sub><b>Phil Miesle</b></sub></a><br /><a href="https://github.com/datastax/cassandra-data-migrator/commits?author=mieslep" title="Code">💻</a></td>
       <td align="center" valign="top" width="16.66%"><a href="https://github.com/mfmaher2"><img src="https://avatars.githubusercontent.com/u/64795956?v=4?s=50" width="50px;" alt="mfmaher2"/><br /><sub><b>mfmaher2</b></sub></a><br /><a href="https://github.com/datastax/cassandra-data-migrator/pulls?q=is%3Apr+reviewed-by%3Amfmaher2" title="Reviewed Pull Requests">👀</a></td>
+      <td align="center" valign="top" width="16.66%"><a href="https://github.com/Jeremya"><img src="https://avatars.githubusercontent.com/u/576519?v=4?s=50" width="50px;" alt="Jeremy"/><br /><sub><b>Jeremy</b></sub></a><br /><a href="https://github.com/datastax/cassandra-data-migrator/commits?author=Jeremya" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

.settings/org.eclipse.jdt.core.prefs

@@ -18,12 +18,17 @@ Migrate and Validate Tables between Origin and Target Cassandra Clusters.
 
 ### Prerequisite
 - Install Java8 as spark binaries are compiled with it.
-- Install Spark version [3.4.1](https://archive.apache.org/dist/spark/spark-3.4.1/) on a single VM (no cluster necessary) where you want to run this job. Spark can be installed by running the following: -
+- Install Spark version [3.4.1](https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz) on a single VM (no cluster necessary) where you want to run this job. Spark can be installed by running the following: -
 ```
 wget https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz
 tar -xvzf spark-3.4.1-bin-hadoop3-scala2.13.tgz
 ```
 
+> :warning: If the above Spark and Scala version is not properly installed, you'll then see a similar exception like below when running the CDM jobs,
+```
+Exception in thread "main" java.lang.NoSuchMethodError: scala.runtime.Statics.releaseFence()V
+```
+
 # Steps for Data-Migration:
 
 > :warning: Note that Version 4 of the tool is not backward-compatible with .properties files created in previous versions, and that package names have changed.
@@ -35,9 +40,9 @@ tar -xvzf spark-3.4.1-bin-hadoop3-scala2.13.tgz
 3. Run the below job using `spark-submit` command as shown below:
 
 ```
-./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
---master "local[*]" --driver-memory 25G --executor-memory 25G /
+./spark-submit --properties-file cdm.properties \
+--conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" \
+--master "local[*]" --driver-memory 25G --executor-memory 25G \
 --class com.datastax.cdm.job.Migrate cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
 
@@ -50,9 +55,9 @@ Note:
 - To run the job in Data validation mode, use class option `--class com.datastax.cdm.job.DiffData` as shown below
 
 ```
-./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
---master "local[*]" --driver-memory 25G --executor-memory 25G /
+./spark-submit --properties-file cdm.properties \
+--conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" \
+--master "local[*]" --driver-memory 25G --executor-memory 25G \
 --class com.datastax.cdm.job.DiffData cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
 
@@ -89,10 +94,10 @@ Note:
 Each line above represents a partition-range (`min,max`). Alternatively, you can also pass the partition-file via command-line param as shown below
 
 ```
-./spark-submit --properties-file cdm.properties /
- --conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
- --conf spark.cdm.tokenRange.partitionFile="/<path-to-file>/<csv-input-filename>" /
---master "local[*]" --driver-memory 25G --executor-memory 25G /
+./spark-submit --properties-file cdm.properties \
+ --conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" \
+ --conf spark.cdm.tokenRange.partitionFile="/<path-to-file>/<csv-input-filename>" \
+ --master "local[*]" --driver-memory 25G --executor-memory 25G \
  --class com.datastax.cdm.job.<Migrate|DiffData> cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```
 This mode is specifically useful to processes a subset of partition-ranges that may have failed during a previous run.
@@ -103,10 +108,10 @@ This mode is specifically useful to processes a subset of partition-ranges that
 # Perform large-field Guardrail violation checks
 - The tool can be used to identify large fields from a table that may break you cluster guardrails (e.g. AstraDB has a 10MB limit for a single large field)  `--class com.datastax.cdm.job.GuardrailCheck` as shown below
 ```
-./spark-submit --properties-file cdm.properties /
---conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" /
---conf spark.cdm.feature.guardrail.colSizeInKB=10000 /
---master "local[*]" --driver-memory 25G --executor-memory 25G /
+./spark-submit --properties-file cdm.properties \
+--conf spark.cdm.schema.origin.keyspaceTable="<keyspacename>.<tablename>" \
+--conf spark.cdm.feature.guardrail.colSizeInKB=10000 \
+--master "local[*]" --driver-memory 25G --executor-memory 25G \
 --class com.datastax.cdm.job.GuardrailCheck cassandra-data-migrator-4.x.x.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt
 ```