Merge pull request datastax#951 from datastax/python-new-row_factory

PYTHON-893: fallback implementation for namedtuple row factory

Author: mambocab

CHANGELOG.rst

@@ -3,6 +3,7 @@
 Bug Fixes
 ---------
 * Tokenmap.get_replicas returns the wrong value if token coincides with the end of the range (PYTHON-978)
+* Python Driver fails with "more than 255 arguments" python exception when > 255 columns specified in query response (PYTHON-893)
 
 Other
 -----

cassandra/query.py

@@ -25,6 +25,7 @@
 import time
 import six
 from six.moves import range, zip
+import warnings
 
 from cassandra import ConsistencyLevel, OperationTimedOut
 from cassandra.util import unix_time_from_uuid1
@@ -83,6 +84,39 @@ def tuple_factory(colnames, rows):
     """
     return rows
 
+class PseudoNamedTupleRow(object):
+    """
+    Helper class for pseudo_named_tuple_factory. These objects provide an
+    __iter__ interface, as well as index- and attribute-based access to values,
+    but otherwise do not attempt to implement the full namedtuple or iterable
+    interface.
+    """
+    def __init__(self, ordered_dict):
+        self._dict = ordered_dict
+        self._tuple = tuple(ordered_dict.values())
+
+    def __getattr__(self, name):
+        return self._dict[name]
+
+    def __getitem__(self, idx):
+        return self._tuple[idx]
+
+    def __iter__(self):
+        return iter(self._tuple)
+
+    def __repr__(self):
+        return '{t}({od})'.format(t=self.__class__.__name__,
+                                  od=self._dict)
+
+
+def pseudo_namedtuple_factory(colnames, rows):
+    """
+    Returns each row as a :class:`.PseudoNamedTupleRow`. This is the fallback
+    factory for cases where :meth:`.named_tuple_factory` fails to create rows.
+    """
+    return [PseudoNamedTupleRow(od)
+            for od in ordered_dict_factory(colnames, rows)]
+
 
 def named_tuple_factory(colnames, rows):
     """
@@ -116,6 +150,20 @@ def named_tuple_factory(colnames, rows):
     clean_column_names = map(_clean_column_name, colnames)
     try:
         Row = namedtuple('Row', clean_column_names)
+    except SyntaxError:
+        warnings.warn(
+            "Failed creating namedtuple for a result because there were too "
+            "many columns. This is due to a Python limitation that affects "
+            "namedtuple in Python 3.0-3.6 (see issue18896). The row will be "
+            "created with {substitute_factory_name}, which lacks some namedtuple "
+            "features and is slower. To avoid slower performance accessing "
+            "values on row objects, Upgrade to Python 3.7, or use a different "
+            "row factory. (column names: {colnames})".format(
+                substitute_factory_name=pseudo_namedtuple_factory.__name__,
+                colnames=colnames
+            )
+        )
+        return pseudo_namedtuple_factory(colnames, rows)
     except Exception:
         clean_column_names = list(map(_clean_column_name, colnames))  # create list because py3 map object will be consumed by first attempt
         log.warning("Failed creating named tuple for results with column names %s (cleaned: %s) "

tests/unit/test_row_factories.py

@@ -0,0 +1,90 @@
+# Copyright DataStax, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from cassandra.query import named_tuple_factory
+
+import logging
+import warnings
+
+import sys
+
+try:
+    from unittest import TestCase
+except ImportError:
+    from unittest2 import TestCase
+
+
+log = logging.getLogger(__name__)
+
+
+NAMEDTUPLE_CREATION_BUG = sys.version_info >= (3,) and sys.version_info < (3, 7)
+
+class TestNamedTupleFactory(TestCase):
+
+    long_colnames, long_rows = (
+        ['col{}'.format(x) for x in range(300)],
+        [
+            ['value{}'.format(x) for x in range(300)]
+            for _ in range(100)
+        ]
+    )
+    short_colnames, short_rows = (
+        ['col{}'.format(x) for x in range(200)],
+        [
+            ['value{}'.format(x) for x in range(200)]
+            for _ in range(100)
+        ]
+    )
+
+    def test_creation_warning_on_long_column_list(self):
+        """
+        Reproduces the failure described in PYTHON-893
+
+        @since 3.15
+        @jira_ticket PYTHON-893
+        @expected_result creation fails on Python > 3 and < 3.7
+
+        @test_category row_factory
+        """
+        if not NAMEDTUPLE_CREATION_BUG:
+            named_tuple_factory(self.long_colnames, self.long_rows)
+            return
+
+        with warnings.catch_warnings(record=True) as w:
+            rows = named_tuple_factory(self.long_colnames, self.long_rows)
+        self.assertEqual(len(w), 1)
+        warning = w[0]
+        self.assertIn('pseudo_namedtuple_factory', str(warning))
+        self.assertIn('3.7', str(warning))
+
+        for r in rows:
+            self.assertEqual(r.col0, self.long_rows[0][0])
+
+    def test_creation_no_warning_on_short_column_list(self):
+        """
+        Tests that normal namedtuple row creation still works after PYTHON-893 fix
+
+        @since 3.15
+        @jira_ticket PYTHON-893
+        @expected_result creates namedtuple-based Rows
+
+        @test_category row_factory
+        """
+        with warnings.catch_warnings(record=True) as w:
+            rows = named_tuple_factory(self.short_colnames, self.short_rows)
+        self.assertEqual(len(w), 0)
+        # check that this is a real namedtuple
+        self.assertTrue(hasattr(rows[0], '_fields'))
+        self.assertIsInstance(rows[0], tuple)