PYTHON-1393 Add support for Cassandra 4.1.x and 5.0 releases to CI (#1220)

Author: absurdfarce

Jenkinsfile

@@ -30,7 +30,7 @@ import com.datastax.jenkins.drivers.python.Slack
 
 slack = new Slack()
 
-DEFAULT_CASSANDRA = ['3.0', '3.11', '4.0']
+DEFAULT_CASSANDRA = ['3.0', '3.11', '4.0', '4.1', '5.0-beta1']
 DEFAULT_DSE = ['dse-5.1.35', 'dse-6.8.30']
 DEFAULT_RUNTIME = ['3.8.16', '3.9.16', '3.10.11', '3.11.3', '3.12.0']
 DEFAULT_CYTHON = ["True", "False"]
@@ -557,6 +557,10 @@ pipeline {
                           <td><strong>4.0</strong></td>
                           <td>Apache CassandraⓇ v4.0.x</td>
                         </tr>
+                        <tr>
+                          <td><strong>5.0-beta1</strong></td>
+                          <td>Apache CassandraⓇ v5.0-beta1</td>
+                        </tr>
                         <tr>
                           <td><strong>dse-5.1.35</strong></td>
                           <td>DataStax Enterprise v5.1.x</td>
@@ -644,7 +648,7 @@ pipeline {
     parameterizedCron(branchPatternCron().matcher(env.BRANCH_NAME).matches() ? """
       # Every weeknight (Monday - Friday) around 4:00 AM
       # These schedules will run with and without Cython enabled for Python 3.8.16 and 3.12.0
-      H 4 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;EVENT_LOOP=LIBEV;CI_SCHEDULE_PYTHON_VERSION=3.8.16 3.12.0;CI_SCHEDULE_SERVER_VERSION=3.11 4.0 dse-5.1.35 dse-6.8.30
+      H 4 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;EVENT_LOOP=LIBEV;CI_SCHEDULE_PYTHON_VERSION=3.8.16 3.12.0;CI_SCHEDULE_SERVER_VERSION=3.11 4.0 5.0-beta1 dse-5.1.35 dse-6.8.30
     """ : "")
   }
 

tests/integration/__init__.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import re
 import os
 from cassandra.cluster import Cluster
 
@@ -561,18 +562,23 @@ def use_cluster(cluster_name, nodes, ipformat=None, start=True, workloads=None,
 
                 CCM_CLUSTER.set_dse_configuration_options(dse_options)
             else:
-                CCM_CLUSTER = CCMCluster(path, cluster_name, **ccm_options)
+                ccm_cluster_clz = CCMCluster if Version(cassandra_version) < Version('4.1') else Cassandra41CCMCluster
+                CCM_CLUSTER = ccm_cluster_clz(path, cluster_name, **ccm_options)
                 CCM_CLUSTER.set_configuration_options({'start_native_transport': True})
                 if Version(cassandra_version) >= Version('2.2'):
                     CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True})
                     if Version(cassandra_version) >= Version('3.0'):
-                        CCM_CLUSTER.set_configuration_options({'enable_scripted_user_defined_functions': True})
-                        if Version(cassandra_version) >= Version('4.0-a'):
+                        # The config.yml option below is deprecated in C* 4.0 per CASSANDRA-17280
+                        if Version(cassandra_version) < Version('4.0'):
+                            CCM_CLUSTER.set_configuration_options({'enable_scripted_user_defined_functions': True})
+                        else:
+                            # Cassandra version >= 4.0
                             CCM_CLUSTER.set_configuration_options({
                                 'enable_materialized_views': True,
                                 'enable_sasi_indexes': True,
                                 'enable_transient_replication': True,
                             })
+
                 common.switch_cluster(path, cluster_name)
                 CCM_CLUSTER.set_configuration_options(configuration_options)
                 CCM_CLUSTER.populate(nodes, ipformat=ipformat, use_single_interface=use_single_interface)
@@ -1011,3 +1017,38 @@ def __new__(cls, **kwargs):
             kwargs['allow_beta_protocol_version'] = cls.DEFAULT_ALLOW_BETA
         return Cluster(**kwargs)
 
+# Subclass of CCMCluster (i.e. ccmlib.cluster.Cluster) which transparently performs
+# conversion of cassandra.yml directives into something matching the new syntax
+# introduced by CASSANDRA-15234
+class Cassandra41CCMCluster(CCMCluster):
+    __test__ = False
+    IN_MS_REGEX = re.compile('^(\w+)_in_ms$')
+    IN_KB_REGEX = re.compile('^(\w+)_in_kb$')
+    ENABLE_REGEX = re.compile('^enable_(\w+)$')
+
+    def _get_config_key(self, k, v):
+        if "." in k:
+            return k
+        m = self.IN_MS_REGEX.match(k)
+        if m:
+            return m.group(1)
+        m = self.ENABLE_REGEX.search(k)
+        if m:
+            return "%s_enabled" % (m.group(1))
+        m = self.IN_KB_REGEX.match(k)
+        if m:
+            return m.group(1)
+        return k
+
+    def _get_config_val(self, k, v):
+        m = self.IN_MS_REGEX.match(k)
+        if m:
+            return "%sms" % (v)
+        m = self.IN_KB_REGEX.match(k)
+        if m:
+            return "%sKiB" % (v)
+        return v
+
+    def set_configuration_options(self, values=None, *args, **kwargs):
+        new_values = {self._get_config_key(k, str(v)):self._get_config_val(k, str(v)) for (k,v) in values.items()}
+        super(Cassandra41CCMCluster, self).set_configuration_options(values=new_values, *args, **kwargs)

tests/integration/cqlengine/query/test_queryset.py

@@ -1362,11 +1362,21 @@ def tearDownClass(cls):
         super(TestModelQueryWithFetchSize, cls).tearDownClass()
         drop_table(TestModelSmall)
 
-    @execute_count(9)
+    @execute_count(19)
     def test_defaultFetchSize(self):
+        # Use smaller batch sizes to avoid hitting the max.  We trigger an InvalidRequest
+        # response for Cassandra 4.1.x and 5.0.x if we just do the whole thing as one
+        # large batch.  We're just using this to populate values for a test, however,
+        # so shifting to smaller batches should be fine.
+        for i in range(0, 5000, 500):
+            with BatchQuery() as b:
+                range_max = i + 500
+                for j in range(i, range_max):
+                    TestModelSmall.batch(b).create(test_id=j)
         with BatchQuery() as b:
-            for i in range(5100):
+            for i in range(5000, 5100):
                 TestModelSmall.batch(b).create(test_id=i)
+
         self.assertEqual(len(TestModelSmall.objects.fetch_size(1)), 5100)
         self.assertEqual(len(TestModelSmall.objects.fetch_size(500)), 5100)
         self.assertEqual(len(TestModelSmall.objects.fetch_size(4999)), 5100)