Integration test: clearer skip/xfail labels

Many tests failing on Scylla were labeled with unconditional xfail.
This has some problems:
- It's hard to tell why a test was marked.
- When some functionality is implemented in Scylla, we don't have an
    easy way to reenable tests that use this functionality.
- Test is skipped also when testing with Cassandra

This commit introduces more labels for failing tests. It fixes those problems:
- Label name and reason string explain why test is disabled
- We can edit label definition to enable tests on newer Scylla version
- Tests are only skipped in environment where they are expected to fail

Author: Lorak-mmk

tests/integration/__init__.py

@@ -344,6 +344,7 @@ def _id_and_mark(f):
 local = local_decorator_creator()
 notprotocolv1 = unittest.skipUnless(PROTOCOL_VERSION > 1, 'Protocol v1 not supported')
 lessthenprotocolv4 = unittest.skipUnless(PROTOCOL_VERSION < 4, 'Protocol versions 4 or greater not supported')
+lessthanprotocolv3 = unittest.skipUnless(PROTOCOL_VERSION < 3, 'Protocol versions 3 or greater not supported')
 greaterthanprotocolv3 = unittest.skipUnless(PROTOCOL_VERSION >= 4, 'Protocol versions less than 4 are not supported')
 protocolv6 = unittest.skipUnless(6 in get_supported_protocol_versions(), 'Protocol versions less than 6 are not supported')
 
@@ -375,6 +376,14 @@ def _id_and_mark(f):
                                               reason='Scylla supports collection indexes from 5.2 onwards') 
 requires_custom_indexes = pytest.mark.xfail(SCYLLA_VERSION is not None, 
                                           reason='Scylla does not support SASI or any other CUSTOM INDEX class')
+requires_java_udf = pytest.mark.xfail(SCYLLA_VERSION is not None,
+                                    reason='Scylla does not support UDFs written in Java')
+requires_composite_type = pytest.mark.xfail(SCYLLA_VERSION is not None,
+                                            reason='Scylla does not support composite types')
+requires_custom_payload = pytest.mark.xfail(SCYLLA_VERSION is not None or PROTOCOL_VERSION < 4,
+                                            reason='Scylla does not support custom payloads. Cassandra requires native protocol v4.0+')
+xfail_scylla = lambda reason, *args, **kwargs: pytest.mark.xfail(SCYLLA_VERSION is not None, reason=reason, *args, **kwargs)
+incorrect_test = lambda reason='This test seems to be incorrect and should be fixed', *args, **kwargs: pytest.mark.xfail(reason=reason, *args, **kwargs)
 
 pypy = unittest.skipUnless(platform.python_implementation() == "PyPy", "Test is skipped unless it's on PyPy")
 notpy3 = unittest.skipIf(sys.version_info >= (3, 0), "Test not applicable for Python 3.x runtime")

tests/integration/standard/test_client_warnings.py

@@ -18,7 +18,8 @@
 import six
 from cassandra.query import BatchStatement
 
-from tests.integration import use_singledc, PROTOCOL_VERSION, local, TestCluster
+from tests.integration import (use_singledc, PROTOCOL_VERSION, local, TestCluster,
+                               requires_custom_payload, xfail_scylla)
 
 
 def setup_module():
@@ -27,7 +28,7 @@ def setup_module():
 
 # Failing with scylla because there is no warning message when changing the value of 'batch_size_warn_threshold_in_kb'
 # config")
-@unittest.expectedFailure
+@xfail_scylla('Empty warnings: TypeError: object of type \'NoneType\' has no len()')
 class ClientWarningTests(unittest.TestCase):
 
     @classmethod
@@ -94,6 +95,7 @@ def test_warning_with_trace(self):
         self.assertIsNotNone(future.get_query_trace())
 
     @local
+    @requires_custom_payload
     def test_warning_with_custom_payload(self):
         """
         Test to validate client warning with custom payload
@@ -113,6 +115,7 @@ def test_warning_with_custom_payload(self):
         self.assertDictEqual(future.custom_payload, payload)
 
     @local
+    @requires_custom_payload
     def test_warning_with_trace_and_custom_payload(self):
         """
         Test to validate client warning with tracing and client warning

tests/integration/standard/test_cluster.py

@@ -42,8 +42,8 @@
 from tests import notwindows
 from tests.integration import use_singledc, get_server_versions, CASSANDRA_VERSION, \
     execute_until_pass, execute_with_long_wait_retry, get_node, MockLoggingHandler, get_unsupported_lower_protocol, \
-    get_unsupported_upper_protocol, protocolv6, local, CASSANDRA_IP, greaterthanorequalcass30, lessthanorequalcass40, \
-    DSE_VERSION, TestCluster, PROTOCOL_VERSION
+    get_unsupported_upper_protocol, lessthanprotocolv3, protocolv6, local, CASSANDRA_IP, greaterthanorequalcass30, \
+    lessthanorequalcass40, DSE_VERSION, TestCluster, PROTOCOL_VERSION, xfail_scylla, incorrect_test
 from tests.integration.util import assert_quiescent_pool_state
 import sys
 
@@ -289,8 +289,7 @@ def test_protocol_negotiation(self):
 
         cluster.shutdown()
 
-    # "Failing with scylla because there is option to create a cluster with 'lower bound' protocol
-    @unittest.expectedFailure
+    @xfail_scylla("Failing with scylla because there is option to create a cluster with 'lower bound' protocol")
     def test_invalid_protocol_negotation(self):
         """
         Test for protocol negotiation when explicit versions are set
@@ -411,12 +410,11 @@ def test_connect_to_bad_hosts(self):
                               protocol_version=PROTOCOL_VERSION)
         self.assertRaises(NoHostAvailable, cluster.connect)
 
+    @lessthanprotocolv3
     def test_cluster_settings(self):
         """
         Test connection setting getters and setters
         """
-        if PROTOCOL_VERSION >= 3:
-            raise unittest.SkipTest("min/max requests and core/max conns aren't used with v3 protocol")
 
         cluster = TestCluster()
 
@@ -1228,8 +1226,7 @@ def test_replicas_are_queried(self):
 
     @greaterthanorequalcass30
     @lessthanorequalcass40
-    # The scylla failed because 'Unknown identifier column1'
-    @unittest.expectedFailure
+    @incorrect_test()
     def test_compact_option(self):
         """
         Test the driver can connect with the no_compact option and the results

tests/integration/standard/test_custom_payload.py

@@ -19,7 +19,8 @@
 
 from cassandra.query import (SimpleStatement, BatchStatement, BatchType)
 
-from tests.integration import use_singledc, PROTOCOL_VERSION, local, TestCluster
+from tests.integration import (use_singledc, PROTOCOL_VERSION, local, TestCluster,
+                              requires_custom_payload)
 
 
 def setup_module():
@@ -28,13 +29,10 @@ def setup_module():
 #These test rely on the custom payload being returned but by default C*
 #ignores all the payloads.
 @local
+@requires_custom_payload
 class CustomPayloadTests(unittest.TestCase):
 
     def setUp(self):
-        if PROTOCOL_VERSION < 4:
-            raise unittest.SkipTest(
-                "Native protocol 4,0+ is required for custom payloads, currently using %r"
-                % (PROTOCOL_VERSION,))
         self.cluster = TestCluster()
         self.session = self.cluster.connect()
 
@@ -43,7 +41,6 @@ def tearDown(self):
         self.cluster.shutdown()
 
     # Scylla error: 'truncated frame: expected 65540 bytes, length is 64'
-    @unittest.expectedFailure
     def test_custom_query_basic(self):
         """
         Test to validate that custom payloads work with simple queries
@@ -67,7 +64,6 @@ def test_custom_query_basic(self):
         self.validate_various_custom_payloads(statement=statement)
 
     # Scylla error: 'Invalid query kind in BATCH messages. Must be 0 or 1 but got 4'"
-    @unittest.expectedFailure
     def test_custom_query_batching(self):
         """
         Test to validate that custom payloads work with batch queries
@@ -94,7 +90,6 @@ def test_custom_query_batching(self):
 
     # Scylla error: 'Got different query ID in server response (b'\x00') than we had before
     # (b'\x84P\xd0K0\xe2=\x11\xba\x02\x16W\xfatN\xf1')'")
-    @unittest.expectedFailure
     def test_custom_query_prepared(self):
         """
         Test to validate that custom payloads work with prepared queries

tests/integration/standard/test_metadata.py

@@ -39,7 +39,8 @@
                                get_supported_protocol_versions, greaterthancass20,
                                greaterthancass21, assert_startswith, greaterthanorequalcass40,
                                greaterthanorequaldse67, lessthancass40,
-                               TestCluster, DSE_VERSION)
+                               TestCluster, DSE_VERSION, requires_java_udf, requires_composite_type,
+                               requires_collection_indexes, xfail_scylla)
 
 from tests.util import wait_until
 
@@ -474,7 +475,7 @@ def test_counter_with_dense_compact_storage(self):
         tablemeta = self.get_table_metadata()
         self.check_create_statement(tablemeta, create_statement)
 
-    @unittest.expectedFailure
+    @xfail_scylla('https://github.com/scylladb/scylladb/issues/6058')
     def test_indexes(self):
         create_statement = self.make_create_statement(["a"], ["b", "c"], ["d", "e", "f"])
         create_statement += " WITH CLUSTERING ORDER BY (b ASC, c ASC)"
@@ -500,7 +501,7 @@ def test_indexes(self):
         self.assertIn('CREATE INDEX e_index', statement)
 
     @greaterthancass21
-    @unittest.expectedFailure
+    @requires_collection_indexes
     def test_collection_indexes(self):
 
         self.session.execute("CREATE TABLE %s.%s (a int PRIMARY KEY, b map<text, text>)"
@@ -530,7 +531,8 @@ def test_collection_indexes(self):
             tablemeta = self.get_table_metadata()
             self.assertIn('(full(b))', tablemeta.export_as_string())
 
-    @unittest.expectedFailure
+    #TODO: Fix Scylla or test
+    @xfail_scylla('Scylla prints `compression = {}` instead of `compression = {\'enabled\': \'false\'}`.')
     def test_compression_disabled(self):
         create_statement = self.make_create_statement(["a"], ["b"], ["c"])
         create_statement += " WITH compression = {}"
@@ -565,7 +567,7 @@ def test_non_size_tiered_compaction(self):
             self.assertNotIn("min_threshold", cql)
             self.assertNotIn("max_threshold", cql)
 
-    @unittest.expectedFailure
+    @requires_java_udf
     def test_refresh_schema_metadata(self):
         """
         test for synchronously refreshing all cluster metadata
@@ -838,7 +840,7 @@ def test_refresh_user_type_metadata_proto_2(self):
             self.assertEqual(cluster.metadata.keyspaces[self.keyspace_name].user_types, {})
             cluster.shutdown()
 
-    @unittest.expectedFailure
+    @requires_java_udf
     def test_refresh_user_function_metadata(self):
         """
         test for synchronously refreshing UDF metadata in keyspace
@@ -875,7 +877,7 @@ def test_refresh_user_function_metadata(self):
 
         cluster2.shutdown()
 
-    @unittest.expectedFailure
+    @requires_java_udf
     def test_refresh_user_aggregate_metadata(self):
         """
         test for synchronously refreshing UDA metadata in keyspace
@@ -919,7 +921,7 @@ def test_refresh_user_aggregate_metadata(self):
         cluster2.shutdown()
 
     @greaterthanorequalcass30
-    @unittest.expectedFailure
+    @requires_collection_indexes
     def test_multiple_indices(self):
         """
         test multiple indices on the same column.
@@ -1544,7 +1546,7 @@ def __init__(self, test_case, **kwargs):
             super(FunctionTest.VerifiedAggregate, self).__init__(test_case, Aggregate, test_case.keyspace_aggregate_meta, **kwargs)
 
 
-@unittest.expectedFailure
+@requires_java_udf
 class FunctionMetadata(FunctionTest):
 
     def make_function_kwargs(self, called_on_null=True):
@@ -1699,6 +1701,7 @@ def test_function_cql_called_on_null(self):
             self.assertRegex(fn_meta.as_cql_query(), "CREATE FUNCTION.*\) RETURNS NULL ON NULL INPUT RETURNS .*")
 
 
+@requires_java_udf
 class AggregateMetadata(FunctionTest):
 
     @classmethod
@@ -1743,7 +1746,6 @@ def make_aggregate_kwargs(self, state_func, state_type, final_func=None, init_co
                 'return_type': "does not matter for creation",
                 'deterministic': False}
 
-    @unittest.expectedFailure
     def test_return_type_meta(self):
         """
         Test to verify to that the return type of a an aggregate is honored in the metadata
@@ -1761,7 +1763,6 @@ def test_return_type_meta(self):
         with self.VerifiedAggregate(self, **self.make_aggregate_kwargs('sum_int', 'int', init_cond='1')) as va:
             self.assertEqual(self.keyspace_aggregate_meta[va.signature].return_type, 'int')
 
-    @unittest.expectedFailure
     def test_init_cond(self):
         """
         Test to verify that various initial conditions are correctly surfaced in various aggregate functions
@@ -1812,7 +1813,6 @@ def test_init_cond(self):
                 self.assertDictContainsSubset(init_not_updated, map_res)
         c.shutdown()
 
-    @unittest.expectedFailure
     def test_aggregates_after_functions(self):
         """
         Test to verify that aggregates are listed after function in metadata
@@ -1835,7 +1835,6 @@ def test_aggregates_after_functions(self):
             self.assertNotIn(-1, (aggregate_idx, func_idx), "AGGREGATE or FUNCTION not found in keyspace_cql: " + keyspace_cql)
             self.assertGreater(aggregate_idx, func_idx)
 
-    @unittest.expectedFailure
     def test_same_name_diff_types(self):
         """
         Test to verify to that aggregates with different signatures are differentiated in metadata
@@ -1858,7 +1857,6 @@ def test_same_name_diff_types(self):
                 self.assertEqual(len(aggregates), 2)
                 self.assertNotEqual(aggregates[0].argument_types, aggregates[1].argument_types)
 
-    @unittest.expectedFailure
     def test_aggregates_follow_keyspace_alter(self):
         """
         Test to verify to that aggregates maintain equality after a keyspace is altered
@@ -1883,7 +1881,6 @@ def test_aggregates_follow_keyspace_alter(self):
             finally:
                 self.session.execute('ALTER KEYSPACE %s WITH durable_writes = true' % self.keyspace_name)
 
-    @unittest.expectedFailure
     def test_cql_optional_params(self):
         """
         Test to verify that the initial_cond and final_func parameters are correctly honored
@@ -2018,7 +2015,7 @@ def test_bad_user_type(self):
             self.assertIn("/*\nWarning:", m.export_as_string())
 
     @greaterthancass21
-    @unittest.expectedFailure
+    @requires_java_udf
     def test_bad_user_function(self):
         self.session.execute("""CREATE FUNCTION IF NOT EXISTS %s (key int, val int)
                                 RETURNS NULL ON NULL INPUT
@@ -2037,7 +2034,7 @@ def test_bad_user_function(self):
                 self.assertIn("/*\nWarning:", m.export_as_string())
 
     @greaterthancass21
-    @unittest.expectedFailure
+    @requires_java_udf
     def test_bad_user_aggregate(self):
         self.session.execute("""CREATE FUNCTION IF NOT EXISTS sum_int (key int, val int)
                                 RETURNS NULL ON NULL INPUT
@@ -2058,7 +2055,7 @@ def test_bad_user_aggregate(self):
 
 class DynamicCompositeTypeTest(BasicSharedKeyspaceUnitTestCase):
 
-    @unittest.expectedFailure
+    @requires_composite_type
     def test_dct_alias(self):
         """
         Tests to make sure DCT's have correct string formatting

tests/integration/standard/test_query.py

@@ -25,7 +25,7 @@
 from cassandra.policies import HostDistance, RoundRobinPolicy, WhiteListRoundRobinPolicy
 from tests.integration import use_singledc, PROTOCOL_VERSION, BasicSharedKeyspaceUnitTestCase, \
     greaterthanprotocolv3, MockLoggingHandler, get_supported_protocol_versions, local, get_cluster, setup_keyspace, \
-    USE_CASS_EXTERNAL, greaterthanorequalcass40, DSE_VERSION, TestCluster, requirecassandra
+    USE_CASS_EXTERNAL, greaterthanorequalcass40, DSE_VERSION, TestCluster, requirecassandra, xfail_scylla
 from tests import notwindows
 from tests.integration import greaterthanorequalcass30, get_node
 
@@ -950,8 +950,7 @@ def test_no_connection_refused_on_timeout(self):
         # Make sure test passed
         self.assertTrue(received_timeout)
 
-    # Failed on Scylla because error `SERIAL/LOCAL_SERIAL consistency may only be requested for one partition at a time`
-    @unittest.expectedFailure
+    @xfail_scylla('Fails on Scylla with error `SERIAL/LOCAL_SERIAL consistency may only be requested for one partition at a time`')
     def test_was_applied_batch_stmt(self):
         """
         Test to ensure `:attr:cassandra.cluster.ResultSet.was_applied` works as expected

tests/integration/standard/test_types.py

@@ -31,7 +31,7 @@
 
 from tests.integration import use_singledc, execute_until_pass, notprotocolv1, \
     BasicSharedKeyspaceUnitTestCase, greaterthancass21, lessthancass30, greaterthanorequaldse51, \
-    DSE_VERSION, greaterthanorequalcass3_10, requiredse, TestCluster
+    DSE_VERSION, greaterthanorequalcass3_10, requiredse, TestCluster, requires_composite_type
 from tests.integration.datatype_utils import update_datatypes, PRIMITIVE_DATATYPES, COLLECTION_TYPES, PRIMITIVE_DATATYPES_KEYS, \
     get_sample, get_all_samples, get_collection_sample
 
@@ -731,7 +731,7 @@ def test_can_insert_unicode_query_string(self):
         s.execute(u"SELECT * FROM system.local WHERE key = 'ef\u2052ef'")
         s.execute(u"SELECT * FROM system.local WHERE key = %s", (u"fe\u2051fe",))
 
-    @unittest.expectedFailure
+    @requires_composite_type
     def test_can_read_composite_type(self):
         """
         Test to ensure that CompositeTypes can be used in a query