Merge branch 'master-private' into ngdg_master_ft

aboudreault · aboudreault · commit 11ac6bb0b954 · 2020-01-09T10:50:43.000-05:00
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -39,6 +39,7 @@ Features
 * Support NULL in collection deserializer (PYTHON-1123)
 * Avoid warnings about unspecified load balancing policy when connecting to a cloud cluster (PYTHON-1177)
 * Add Python 3.8 support (PYTHON-1189)
+* Add new DSE CQL keywords (PYTHON-1122)
 * [GRAPH] Ability to execute Fluent Graph queries asynchronously (PYTHON-1129)
 
 Bug Fixes
@@ -48,9 +49,12 @@ Bug Fixes
 * Reconnect attempts persist after downed node removed from peers (PYTHON-1181)
 * Connection fails to validate ssl certificate hostname when SSLContext.check_hostname is set (PYTHON-1186)
 * ResponseFuture._set_result crashes on connection error when used with PrepareMessage (PYTHON-1187)
+* Insights fail to serialize the startup message when the SSL Context is from PyOpenSSL (PYTHON-1192)
 
 Others
 ------
+* The driver has a new dependency: geomet. It comes from the dse-driver unification and
+  is used to support DSE geo types.
 
 Deprecations
 ------------
diff --git a/README.rst b/README.rst
@@ -1,5 +1,5 @@
-DataStax Python Driver for Apache Cassandra and DataStax products
-=================================================================
+DataStax Driver for Apache Cassandra
+====================================
 
 .. image:: https://travis-ci.org/datastax/python-driver.png?branch=master
    :target: https://travis-ci.org/datastax/python-driver
diff --git a/build.yaml b/build.yaml
@@ -153,19 +153,6 @@ schedules:
         - python: [3.4, 3.6, 3.7, 3.8]
         - cassandra: ['2.0', '2.1', '2.2', '3.0', 'test-dse']
 
-  # TODO: Temporary, don't merge me
-  python_oss_next_tests:
-    schedule: adhoc
-    disable_pull_requests: true
-    branches:
-      include: [/oss-next.*/]
-    env_vars: |
-      EVENT_LOOP_MANAGER='libev'
-      EXCLUDE_LONG=1
-    matrix:
-      exclude:
-        - python: [3.4, 3.6, 3.7, 3.8]
-
 python:
   - 2.7
   - 3.4
@@ -192,7 +179,7 @@ cassandra:
 env:
   CYTHON:
     - CYTHON
-#    - NO_CYTHON
+    - NO_CYTHON
 
 build:
   - script: |
@@ -210,7 +197,12 @@ build:
 
       pip install git+ssh://git@github.com/riptano/ccm-private.git
 
-      pip install -r test-requirements.txt
+      if [ -n "$CCM_IS_DSE" ]; then
+        pip install -r test-datastax-requirements.txt
+      else
+        pip install -r test-requirements.txt
+      fi
+
       pip install nose-ignore-docstring
       pip install nose-exclude
       pip install service_identity
@@ -277,14 +269,6 @@ build:
       echo "==========RUNNING INTEGRATION TESTS=========="
       EVENT_LOOP_MANAGER=$EVENT_LOOP_MANAGER CCM_ARGS="$CCM_ARGS" DSE_VERSION=$DSE_VERSION CASSANDRA_VERSION=$CCM_CASSANDRA_VERSION MAPPED_CASSANDRA_VERSION=$MAPPED_CASSANDRA_VERSION VERIFY_CYTHON=$FORCE_CYTHON nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml tests/integration/standard/ || true
 
-      # TODO: Temporary, don't merge this 2.1 stuff...
-      # if [ "$CCM_CASSANDRA_VERSION" == "2.1.21" ]; then
-      #   echo "Testing cassandra 2.1 with protocol V2"
-      #   PROTOCOL_VERSION=2 EVENT_LOOP_MANAGER=$EVENT_LOOP_MANAGER CCM_ARGS="$CCM_ARGS" CASSANDRA_VERSION=$CCM_CASSANDRA_VERSION MAPPED_CASSANDRA_VERSION=$MAPPED_CASSANDRA_VERSION VERIFY_CYTHON=$FORCE_CYTHON nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_v2_results.xml tests/integration/standard/ || true
-      #   echo "Testing cassandra 2.1 with protocol V1"
-      #   PROTOCOL_VERSION=1 EVENT_LOOP_MANAGER=$EVENT_LOOP_MANAGER CCM_ARGS="$CCM_ARGS" CASSANDRA_VERSION=$CCM_CASSANDRA_VERSION MAPPED_CASSANDRA_VERSION=$MAPPED_CASSANDRA_VERSION VERIFY_CYTHON=$FORCE_CYTHON nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_v1_results.xml tests/integration/standard/ || true
-      # fi
-
       if [ -n "$DSE_VERSION" ] && ! [[ $DSE_VERSION == "4.8"* ]]; then
         echo "==========RUNNING DSE INTEGRATION TESTS=========="
         EVENT_LOOP_MANAGER=$EVENT_LOOP_MANAGER CASSANDRA_DIR=$CCM_INSTALL_DIR DSE_VERSION=$DSE_VERSION ADS_HOME=$HOME/ VERIFY_CYTHON=$FORCE_CYTHON nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=dse_results.xml tests/integration/advanced/ || true
diff --git a/cassandra/__init__.py b/cassandra/__init__.py
@@ -191,7 +191,7 @@ class ProtocolVersion(object):
 
     MAX_SUPPORTED = max(SUPPORTED_VERSIONS)
     """
-    Maximum protocol versioni supported by this driver.
+    Maximum protocol version supported by this driver.
     """
 
     @classmethod
diff --git a/cassandra/auth.py b/cassandra/auth.py
@@ -150,8 +150,8 @@ class TransitionalModePlainTextAuthProvider(object):
         auth_provider = TransitionalModePlainTextAuthProvider()
         cluster = Cluster(auth_provider=auth_provider)
 
-    .. warning:: TransitionalModePlainTextAuthProvider will be removed in dse-driver
-                 3.0. The transitional mode will be handled internally without the need
+    .. warning:: TransitionalModePlainTextAuthProvider will be removed in cassandra-driver
+                 4.0. The transitional mode will be handled internally without the need
                  of any auth provider.
     """
 
diff --git a/cassandra/cluster.py b/cassandra/cluster.py
@@ -450,7 +450,7 @@ def __init__(self, load_balancing_policy=None, retry_policy=None,
         See also :class:`~.GraphExecutionPolicy`.
 
         In addition to default parameters shown in the signature, this profile also defaults ``retry_policy`` to
-        :class:`dse.policies.NeverRetryPolicy`, and ``load_balancing_policy`` to one that targets the current Spark
+        :class:`cassandra.policies.NeverRetryPolicy`, and ``load_balancing_policy`` to one that targets the current Spark
         master.
 
         Note: The graph_options.graph_source is set automatically to b'a' (analytics)
@@ -3055,7 +3055,7 @@ def prepare(self, query, custom_payload=None, keyspace=None):
         future = ResponseFuture(self, message, query=None, timeout=self.default_timeout)
         try:
             future.send_request()
-            response = future.result()[0]
+            response = future.result().one()
         except Exception:
             log.exception("Error preparing query:")
             raise
diff --git a/cassandra/datastax/insights/reporter.py b/cassandra/datastax/insights/reporter.py
@@ -135,14 +135,23 @@ def _get_startup_data(self):
             'ignored': host_distances_counter[HostDistance.IGNORED]
         }
 
-        compression_type = cc._connection._compression_type if cc._connection else 'NONE'
+        try:
+            compression_type = cc._connection._compression_type
+        except AttributeError:
+            compression_type = 'NONE'
 
-        if self._session.cluster.ssl_context:
-            cert_validation = self._session.cluster.ssl_context.verify_mode == ssl.CERT_REQUIRED
-        elif self._session.cluster.ssl_options:
-            cert_validation = self._session.cluster.ssl_options.get('cert_reqs') == ssl.CERT_REQUIRED
-        else:
-            cert_validation = None
+        cert_validation = None
+        try:
+            if self._session.cluster.ssl_context:
+                if isinstance(self._session.cluster.ssl_context, ssl.SSLContext):
+                    cert_validation = self._session.cluster.ssl_context.verify_mode == ssl.CERT_REQUIRED
+                else:  # pyopenssl
+                    from OpenSSL import SSL
+                    cert_validation = self._session.cluster.ssl_context.get_verify_mode() != SSL.VERIFY_NONE
+            elif self._session.cluster.ssl_options:
+                cert_validation = self._session.cluster.ssl_options.get('cert_reqs') == ssl.CERT_REQUIRED
+        except Exception as e:
+            log.debug('Unable to get the cert validation: {}'.format(e))
 
         uname_info = platform.uname()
 
@@ -157,7 +166,7 @@ def _get_startup_data(self):
                 },
             },
             'data': {
-                'driverName': 'DataStax Enterprise Python Driver',
+                'driverName': 'DataStax Python Driver',
                 'driverVersion': sys.modules['cassandra'].__version__,
                 'clientId': str(self._session.cluster.client_id),
                 'sessionId': str(self._session.session_id),
diff --git a/cassandra/metadata.py b/cassandra/metadata.py
@@ -57,7 +57,14 @@
     'permissions', 'primary', 'rename', 'replace', 'returns', 'revoke', 'role', 'roles', 'schema', 'select', 'set',
     'sfunc', 'smallint', 'static', 'storage', 'stype', 'superuser', 'table', 'text', 'time', 'timestamp', 'timeuuid',
     'tinyint', 'to', 'token', 'trigger', 'truncate', 'ttl', 'tuple', 'type', 'unlogged', 'update', 'use', 'user',
-    'users', 'using', 'uuid', 'values', 'varchar', 'varint', 'view', 'where', 'with', 'writetime'
+    'users', 'using', 'uuid', 'values', 'varchar', 'varint', 'view', 'where', 'with', 'writetime',
+
+    # DSE specifics
+    "node", "nodes", "plan", "active", "application", "applications", "java", "executor", "executors", "std_out", "std_err",
+    "renew", "delegation", "no", "redact", "token", "lowercasestring", "cluster", "authentication", "schemes", "scheme",
+    "internal", "ldap", "kerberos", "remote", "object", "method", "call", "calls", "search", "schema", "config", "rows",
+    "columns", "profiles", "commit", "reload", "unset", "rebuild", "field", "workpool", "any", "submission", "indices",
+    "restrict", "unrestrict"
 ))
 """
 Set of keywords in CQL.
diff --git a/cassandra/policies.py b/cassandra/policies.py
@@ -1168,8 +1168,8 @@ class DSELoadBalancingPolicy(DefaultLoadBalancingPolicy):
     *Deprecated:* This will be removed in the next major release,
     consider using :class:`.DefaultLoadBalancingPolicy`.
     """
-    def __init__(self, *args):
-        super(DSELoadBalancingPolicy, self).__init__(*args)
+    def __init__(self, *args, **kwargs):
+        super(DSELoadBalancingPolicy, self).__init__(*args, **kwargs)
         warnings.warn("DSELoadBalancingPolicy will be removed in 4.0. Consider using "
                       "DefaultLoadBalancingPolicy.", DeprecationWarning)
 
diff --git a/cassandra/protocol.py b/cassandra/protocol.py
@@ -589,7 +589,7 @@ def _write_query_params(self, f, protocol_version):
             else:
                 raise UnsupportedOperation(
                     "Keyspaces may only be set on queries with protocol version "
-                    "DSE_V2 or higher. Consider setting Cluster.protocol_version to ProtocolVersion.DSE_V2.")
+                    "5 or DSE_V2 or higher. Consider setting Cluster.protocol_version.")
 
         if ProtocolVersion.uses_int_query_flags(protocol_version):
             write_uint(f, flags)
diff --git a/docs/getting_started.rst b/docs/getting_started.rst
@@ -47,7 +47,7 @@ behavior in some other way, this is the place to do it:
     cluster = Cluster(execution_profiles={EXEC_PROFILE_DEFAULT: profile})
     session = cluster.connect()
 
-    print session.execute("SELECT release_version FROM system.local")[0]
+    print(session.execute("SELECT release_version FROM system.local").one())
 
 Profiles are passed in by ``execution_profiles`` dict.
 
@@ -70,7 +70,7 @@ In this case we can construct the base ``ExecutionProfile`` passing all attribut
     cluster = Cluster(execution_profiles={EXEC_PROFILE_DEFAULT: profile})
     session = cluster.connect()
 
-    print session.execute("SELECT release_version FROM system.local")[0]
+    print(session.execute("SELECT release_version FROM system.local").one())
 
 Users are free to setup additional profiles to be used by name:
 
diff --git a/docs/index.rst b/docs/index.rst
@@ -1,7 +1,7 @@
-DataStax Python Driver for Apache Cassandra and DataStax products
-=================================================================
-A Python client driver for `Apache Cassandra <http://cassandra.apache.org>`_ and
-DataStax products. This driver works exclusively with the Cassandra Query Language v3 (CQL3)
+DataStax Python Driver for Apache Cassandra
+===========================================
+A Python client driver for `Apache Cassandra <http://cassandra.apache.org>`_.
+This driver works exclusively with the Cassandra Query Language v3 (CQL3)
 and Cassandra's native protocol.  Cassandra 2.1+ is supported, including DSE 4.7+.
 
 The driver supports Python 2.7, 3.4, 3.5, 3.6, 3.7 and 3.8.
diff --git a/setup.py b/setup.py
@@ -416,7 +416,7 @@ def run_setup(extensions):
     setup(
         name='cassandra-driver',
         version=__version__,
-        description=' DataStax Driver for Apache Cassandra and DataStax products',
+        description=' DataStax Driver for Apache Cassandra',
         long_description=long_description,
         url='http://github.com/datastax/python-driver',
         project_urls={
diff --git a/test-datastax-requirements.txt b/test-datastax-requirements.txt
@@ -0,0 +1,3 @@
+-r test-requirements.txt
+kerberos
+gremlinpython==3.3.4
diff --git a/test-requirements.txt b/test-requirements.txt
@@ -13,10 +13,7 @@ gevent>=1.0
 eventlet
 cython>=0.20,<0.30
 packaging
-kerberos
-requests
 backports.ssl_match_hostname; python_version < '2.7.9'
 futurist; python_version >= '3.7'
 asynctest; python_version > '3.4'
-gremlinpython==3.3.4
 ipaddress; python_version < '3.3.0'
diff --git a/tests/integration/advanced/graph/fluent/test_graph.py b/tests/integration/advanced/graph/fluent/test_graph.py
@@ -645,7 +645,7 @@ def _validate_path_result_type(self, g, objects_path):
             elif isinstance(obj, TravVertex):
                 self._validate_classic_vertex(g, obj)
             else:
-                self.fail("Invalid object found in path " + str(object.type))
+                self.fail("Invalid object found in path " + str(obj.type))
 
     def _validate_meta_property(self, g, vertex):
         meta_props = g.V(vertex.id).properties().toList()
diff --git a/tests/integration/simulacron/test_policies.py b/tests/integration/simulacron/test_policies.py
@@ -17,9 +17,10 @@
     import unittest  # noqa
 
 from cassandra import OperationTimedOut, WriteTimeout
-from cassandra.cluster import Cluster, ExecutionProfile, ResponseFuture, EXEC_PROFILE_DEFAULT
+from cassandra.cluster import Cluster, ExecutionProfile, ResponseFuture, EXEC_PROFILE_DEFAULT, NoHostAvailable
 from cassandra.query import SimpleStatement
 from cassandra.policies import ConstantSpeculativeExecutionPolicy, RoundRobinPolicy, RetryPolicy, WriteType
+from cassandra.protocol import OverloadedErrorMessage, IsBootstrappingErrorMessage, TruncateError, ServerError
 
 from tests.integration import greaterthancass21, requiressimulacron, SIMULACRON_JAR, \
     CASSANDRA_VERSION
@@ -242,7 +243,7 @@ def on_unavailable(self, query, consistency, required_replicas, alive_replicas,
 
     def on_request_error(self, query, consistency, error, retry_num):
         next(self.request_error)
-        return self.IGNORE, None
+        return self.RETHROW, None
 
     def reset_counters(self):
         self.write_timeout = count()
@@ -429,14 +430,20 @@ def test_retry_policy_on_request_error(self):
         retry_policy = CounterRetryPolicy()
         self.set_cluster(retry_policy)
 
-        for e in [overloaded_error, bootstrapping_error, truncate_error, server_error]:
+        for prime_error, exc in [
+            (overloaded_error, OverloadedErrorMessage),
+            (bootstrapping_error, IsBootstrappingErrorMessage),
+            (truncate_error, TruncateError),
+            (server_error, ServerError)]:
+
+            clear_queries()
             query_to_prime = "SELECT * from simulacron_keyspace.simulacron_table;"
-            prime_query(query_to_prime, then=e, rows=None, column_types=None)
+            prime_query(query_to_prime, then=prime_error, rows=None, column_types=None)
             rf = self.session.execute_async(query_to_prime)
-            try:
+
+            with self.assertRaises(exc):
                 rf.result()
-            except:
-                pass
+
             self.assertEqual(len(rf.attempted_hosts), 1)  # no retry
 
         self.assertEqual(next(retry_policy.request_error), 4)
@@ -446,11 +453,12 @@ def test_retry_policy_on_request_error(self):
         self.set_cluster(retry_policy)
 
         for e in [overloaded_error, bootstrapping_error, truncate_error, server_error]:
+            clear_queries()
             query_to_prime = "SELECT * from simulacron_keyspace.simulacron_table;"
             prime_query(query_to_prime, then=e, rows=None, column_types=None)
             rf = self.session.execute_async(query_to_prime)
-            try:
+
+            with self.assertRaises(NoHostAvailable):
                 rf.result()
-            except:
-                pass
+
             self.assertEqual(len(rf.attempted_hosts), 3)  # all 3 nodes failed
diff --git a/tests/unit/advanced/test_auth.py b/tests/unit/advanced/test_auth.py
@@ -21,7 +21,10 @@
 
 from cassandra.auth import DSEGSSAPIAuthProvider
 
+from tests.integration import requiredse
 
+# This auth provider requires kerberos and puresals
+@requiredse
 class TestGSSAPI(unittest.TestCase):
 
     def test_host_resolution(self):
