SPARKC-657 Bump DSE & OSS C* versions to current patch/RC releases (#1321)

Allow for DSE 5.1 series to be considered as DSE version
Protocol v5 only support LZ4 compression

Co-authored-by: Madhavan Sridharan <[email protected]>

Author: msmygit

.github/workflows/main.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         scala:  [2.11.12]
-        db-version: [6.8.9, 3.11.9, 4.0-beta4]
+        db-version: [6.8.13, 5.1.24, 3.11.10, 4.0-rc2]
 
     steps:
       - uses: actions/checkout@v2
@@ -35,4 +35,4 @@ jobs:
           TEST_PARALLEL_TASKS: 1
           CCM_CASSANDRA_VERSION: ${{ matrix.db-version }}
           PUBLISH_VERSION: test
-        run: sbt/sbt ++${{ matrix.scala }} test it:test
+        run: sbt/sbt ++${{ matrix.scala }} test it:test

connector/src/it/scala/com/datastax/spark/connector/cql/CassandraConnectorSpec.scala

@@ -4,6 +4,7 @@ import java.io.IOException
 
 import com.datastax.oss.driver.api.core.config.DefaultDriverOption
 import com.datastax.oss.driver.api.core.cql.SimpleStatement
+import com.datastax.oss.driver.api.core.{ProtocolVersion, Version}
 import com.datastax.spark.connector.cluster.DefaultCluster
 import com.datastax.spark.connector.{SparkCassandraITFlatSpecBase, _}
 import org.scalatest.BeforeAndAfterEach
@@ -165,7 +166,7 @@ class CassandraConnectorSpec extends SparkCassandraITFlatSpecBase with DefaultCl
     conn2.withSessionDo { session => }
   }
 
-  it should "use compression when configured" in {
+  it should "use SNAPPY compression when configured" in skipIfProtocolVersionGTE(ProtocolVersion.V5) {
     val conf = sc.getConf
       .set(CassandraConnectorConf.CompressionParam.name, "SNAPPY")
 
@@ -174,7 +175,21 @@ class CassandraConnectorSpec extends SparkCassandraITFlatSpecBase with DefaultCl
       session
         .getContext
         .getConfig
-        .getDefaultProfile.getString(DefaultDriverOption.PROTOCOL_COMPRESSION) shouldBe  "snappy"
+        .getDefaultProfile.getString(DefaultDriverOption.PROTOCOL_COMPRESSION) shouldBe "snappy"
+    }
+  }
+
+  // https://github.com/apache/cassandra/blob/cassandra-4.0.0/doc/native_protocol_v5.spec#L194-L195
+  it should "use LZ4 compression when configured" in skipIfProtocolVersionLT(ProtocolVersion.V5) {
+    val conf = sc.getConf
+      .set(CassandraConnectorConf.CompressionParam.name, "LZ4")
+
+    val conn = CassandraConnector(conf)
+    conn.withSessionDo { session =>
+      session
+        .getContext
+        .getConfig
+        .getDefaultProfile.getString(DefaultDriverOption.PROTOCOL_COMPRESSION) shouldBe "lz4"
     }
   }
 

doc/developers.md

@@ -22,7 +22,7 @@ The most common commands to use when developing the connector are
 1. `test` - Runs the the unit tests for the project.
 2. `it:test` - Runs the integration tests with Cassandra (started by CCM) and Spark
 3. `package` - Builds the project and produces a runtime jar
-4. `publishM2` - Publishes a snapshot of the project to your local maven repository allowing for usage with --packages in the spark-shell
+4. `publishM2` - Publishes a snapshot of the project to your local maven repository allowing for usage with `--packages` in the spark-shell
 
 The integration tests located in `connector/src/it` should
 probably be the first place to look for anyone considering adding code.
@@ -39,7 +39,7 @@ b2.5 feature branch to b3.0 feature branch. Repeat for master.
 
 Example for imaginary SPARKC-9999.
 
-Let's assume that `datastax` is [email protected]:datastax/spark-cassandra-connector.git remote 
+Let's assume that `datastax` is `[email protected]:datastax/spark-cassandra-connector.git` remote 
 and origin is your personal clone.
 ```shell
 $ git remote -v
@@ -95,11 +95,11 @@ run and the parallelization used while running tests.
 ### Test Parallelization
 
 In order to limit the number of test groups running simultaneously use the
-`TEST_PARALLEL_TASKS` environment variable. Only applies to sbt test tasks.
+`TEST_PARALLEL_TASKS` environment variable. Only applies to `sbt test` tasks.
 
 ### Set Cassandra Test Target
 Our CI Build runs through the Datastax Infrastructure and tests on all the builds
-listed in build.yaml. In addition the test-support module supports Cassandra
+listed in build.yaml. In addition the _test-support_ module supports Cassandra
 or other CCM Compatible installations.
 
 If using SBT you can set

project/Testing.scala

@@ -45,11 +45,11 @@ object Testing {
 
   /**
     * Guesses if the given version is DSE or not.
-    * Internally it checks if the major version component is ge 6 which should suffice for a long time.
+    * Internally it checks if the major version component is ge 5 which should suffice for a long time.
     * CCM_IS_DSE env setting takes precedence over this guess.
     */
   private def isDse(version: Option[String]): Boolean = {
-    version.flatMap(v => Try(v.split('.').head.toInt >= 6).toOption).getOrElse(false)
+    version.flatMap(v => Try(v.split('.').head.toInt >= 5).toOption).getOrElse(false)
   }
 
   def getCCMJvmOptions = {