Merge pull request datastax#985 from datastax/long-python_test-dse-failures

Long python test dse failures

Author: mambocab

CHANGELOG.rst

@@ -1,3 +1,10 @@
+3.15.2
+======
+
+Other
+-----
+* Fix tests when RF is not maintained if we decomission a node (PYTHON-1017)
+
 3.15.1
 ======
 September 6, 2018

build.yaml

@@ -20,6 +20,17 @@ schedules:
       exclude:
         - python: [3.4, 3.6]
 
+  commit_long_test:
+    schedule: per_commit
+    branches:
+      include: [/long-python.*/]
+    env_vars: |
+      EVENT_LOOP_MANAGER='libev'
+    matrix:
+      exclude:
+        - python: [3.4, 3.6]
+        - cassandra: ['2.0', '2.1', '3.0']
+
   commit_branches:
     schedule: per_commit
     branches:

tests/integration/long/test_loadbalancingpolicies.py

@@ -65,13 +65,20 @@ def _connect_probe_cluster(self):
             self.probe_session = self.probe_cluster.connect()
 
     def _wait_for_nodes_up(self, nodes, cluster=None):
+        log.debug('entered: _wait_for_nodes_up(nodes={ns}, '
+                  'cluster={cs})'.format(ns=nodes,
+                                         cs=cluster))
         if not cluster:
+            log.debug('connecting to cluster')
             self._connect_probe_cluster()
             cluster = self.probe_cluster
         for n in nodes:
             wait_for_up(cluster, n)
 
     def _wait_for_nodes_down(self, nodes, cluster=None):
+        log.debug('entered: _wait_for_nodes_down(nodes={ns}, '
+                  'cluster={cs})'.format(ns=nodes,
+                                         cs=cluster))
         if not cluster:
             self._connect_probe_cluster()
             cluster = self.probe_cluster
@@ -87,13 +94,19 @@ def _cluster_session_with_lbp(self, lbp):
 
     def _insert(self, session, keyspace, count=12,
                 consistency_level=ConsistencyLevel.ONE):
+        log.debug('entered _insert('
+                  'session={session}, keyspace={keyspace}, '
+                  'count={count}, consistency_level={consistency_level}'
+                  ')'.format(session=session, keyspace=keyspace, count=count,
+                             consistency_level=consistency_level))
         session.execute('USE %s' % keyspace)
         ss = SimpleStatement('INSERT INTO cf(k, i) VALUES (0, 0)', consistency_level=consistency_level)
 
         tries = 0
         while tries < 100:
             try:
                 execute_concurrent_with_args(session, ss, [None] * count)
+                log.debug('Completed _insert on try #{}'.format(tries + 1))
                 return
             except (OperationTimedOut, WriteTimeout, WriteFailure):
                 ex_type, ex, tb = sys.exc_info()
@@ -105,6 +118,13 @@ def _insert(self, session, keyspace, count=12,
 
     def _query(self, session, keyspace, count=12,
                consistency_level=ConsistencyLevel.ONE, use_prepared=False):
+        log.debug('entered _query('
+                  'session={session}, keyspace={keyspace}, '
+                  'count={count}, consistency_level={consistency_level}, '
+                  'use_prepared={use_prepared}'
+                  ')'.format(session=session, keyspace=keyspace, count=count,
+                             consistency_level=consistency_level,
+                             use_prepared=use_prepared))
         if use_prepared:
             query_string = 'SELECT * FROM %s.cf WHERE k = ?' % keyspace
             if not self.prepared or self.prepared.query_string != query_string:
@@ -549,7 +569,7 @@ def test_token_aware_with_shuffle_rf2(self):
 
         self._check_query_order_changes(session=session, keyspace=keyspace)
 
-        #check TokenAwarePolicy still return the remaining replicas when one goes down
+        # check TokenAwarePolicy still return the remaining replicas when one goes down
         self.coordinator_stats.reset_counts()
         stop(2)
         self._wait_for_nodes_down([2], cluster)

tests/integration/long/utils.py

@@ -17,8 +17,9 @@
 import time
 
 from collections import defaultdict
-from ccmlib.node import Node
+from ccmlib.node import Node, ToolError
 
+from nose.tools import assert_in
 from cassandra.query import named_tuple_factory
 from cassandra.cluster import ConsistencyLevel
 
@@ -35,6 +36,7 @@ def __init__(self):
         self.coordinator_counts = defaultdict(int)
 
     def add_coordinator(self, future):
+        log.debug('adding coordinator from {}'.format(future))
         future.result()
         coordinator = future._current_host.address
         self.coordinator_counts[coordinator] += 1
@@ -100,11 +102,24 @@ def force_stop(node):
 
 
 def decommission(node):
-    get_node(node).decommission()
+    try:
+        get_node(node).decommission()
+    except ToolError as e:
+        expected_errs = (('Not enough live nodes to maintain replication '
+                          'factor in keyspace system_distributed'),
+                         'Perform a forceful decommission to ignore.')
+        for err in expected_errs:
+            assert_in(err, e.stdout)
+        # in this case, we're running against a C* version with CASSANDRA-12510
+        # applied and need to decommission with `--force`
+        get_node(node).decommission(force=True)
     get_node(node).stop()
 
 
 def bootstrap(node, data_center=None, token=None):
+    log.debug('called bootstrap('
+              'node={node}, data_center={data_center}, '
+              'token={token})')
     node_instance = Node('node%s' % node,
                          get_cluster(),
                          auto_bootstrap=False,
@@ -118,12 +133,15 @@ def bootstrap(node, data_center=None, token=None):
 
     try:
         start(node)
-    except:
+    except Exception as e0:
+        log.debug('failed 1st bootstrap attempt with: \n{}'.format(e0))
         # Try only twice
         try:
             start(node)
-        except:
+        except Exception as e1:
+            log.debug('failed 2nd bootstrap attempt with: \n{}'.format(e1))
             log.error('Added node failed to start twice.')
+            raise e1
 
 
 def ring(node):
@@ -140,7 +158,7 @@ def wait_for_up(cluster, node):
             log.debug("Done waiting for node %s to be up", node)
             return
         else:
-            log.debug("Host is still marked down, waiting")
+            log.debug("Host {} is still marked down, waiting".format(addr))
             tries += 1
             time.sleep(1)