SNOW-27306: removed generator and enumerate from row_to_python method to improve fetch performance. This change made fetch 60% faster.

Author: smtakeda

converter.py

@@ -89,18 +89,21 @@ def _generate_tzinfo_from_tzoffset(self, tzoffset_minutes):
     # FROM Snowflake to Python Objects
     #
     def to_python_method(self, type_name, row_type):
+        ctx = {
+            u'scale': row_type['scale'],
+        }
         try:
             if self._use_numpy:
                 return getattr(self, u'_{type_name}_numpy_to_python'.format(
-                    type_name=type_name)), None
+                    type_name=type_name)), ctx
             elif type_name == 'FIXED' and row_type['scale'] == 0:
                 return self._FIXED_INT_to_python, None
             else:
                 return getattr(self, u'_{type_name}_to_python'.format(
-                    type_name=type_name)), None
+                    type_name=type_name)), ctx
         except KeyError:
             # no type is defined
-            return self._str_to_snowflake, None
+            return self._TEXT_to_python, None
 
     def _FIXED_INT_to_python(self, value, *_):
         return int(value)
@@ -144,10 +147,10 @@ def _DATE_numpy_to_python(self, value, *_):
         """
         return numpy.datetime64(int(value), 'D')
 
-    def _extract_timestamp(self, value, col_desc, has_tz=False):
+    def _extract_timestamp(self, value, ctx, has_tz=False):
         """Extracts timstamp from a raw data
         """
-        scale = col_desc[5]
+        scale = ctx['scale']
         try:
             value1 = decimal.Decimal(value)
             big_int = int(value1.scaleb(scale))  # removed fraction
@@ -167,7 +170,7 @@ def _extract_timestamp(self, value, col_desc, has_tz=False):
         except decimal.InvalidOperation:
             return None, None, None
 
-    def _pre_TIMESTAMP_TZ_to_python(self, value, col_desc):
+    def _pre_TIMESTAMP_TZ_to_python(self, value, ctx):
         u"""
         try to split value by space for handling new timestamp with timezone
         encoding format which has timezone index separate from the timestamp
@@ -182,7 +185,7 @@ def _pre_TIMESTAMP_TZ_to_python(self, value, col_desc):
             value = valueComponents[0]
 
         tzoffset, microseconds, fraction_of_nanoseconds, nanoseconds = \
-            self._extract_timestamp(value, col_desc,
+            self._extract_timestamp(value, ctx,
                                     has_tz=(tzoffset_extracted is None))
 
         if tzoffset_extracted is not None:
@@ -198,34 +201,34 @@ def _pre_TIMESTAMP_TZ_to_python(self, value, col_desc):
             t += tzinfo_value.utcoffset(t, is_dst=False)
         return t.replace(tzinfo=tzinfo_value), fraction_of_nanoseconds
 
-    def _TIMESTAMP_TZ_to_python(self, value, col_desc, *_):
+    def _TIMESTAMP_TZ_to_python(self, value, ctx):
         """
         TIMESTAMP TZ to datetime
 
         The timezone offset is piggybacked.
         """
-        t, _ = self._pre_TIMESTAMP_TZ_to_python(value, col_desc)
+        t, _ = self._pre_TIMESTAMP_TZ_to_python(value, ctx)
         return t
 
-    def _TIMESTAMP_TZ_numpy_to_python(self, value, col_desc, *_):
+    def _TIMESTAMP_TZ_numpy_to_python(self, value, ctx):
         """TIMESTAMP TZ to datetime
 
         The timezone offset is piggybacked.
         """
         t, fraction_of_nanoseconds = self._pre_TIMESTAMP_TZ_to_python(
-            value, col_desc)
+            value, ctx)
         ts = int(time.mktime(t.timetuple())) * 1000000000 + int(
             fraction_of_nanoseconds)
         return numpy.datetime64(ts, 'ns')
 
-    def _pre_TIMESTAMP_LTZ_to_python(self, value, col_desc):
+    def _pre_TIMESTAMP_LTZ_to_python(self, value, ctx):
         u""" TIMESTAMP LTZ to datetime
 
         This takes consideration of the session parameter TIMEZONE if
         available. If not, tzlocal is used
         """
         tzoffset, microseconds, fraction_of_nanoseconds, nanoseconds = \
-            self._extract_timestamp(value, col_desc)
+            self._extract_timestamp(value, ctx)
         if tzoffset is None:
             return None
         try:
@@ -251,39 +254,39 @@ def _pre_TIMESTAMP_LTZ_to_python(self, value, col_desc):
             t = time.gmtime(microseconds / float(1000000))
             return t, fraction_of_nanoseconds
 
-    def _TIMESTAMP_LTZ_to_python(self, value, col_desc, *_):
-        t, _ = self._pre_TIMESTAMP_LTZ_to_python(value, col_desc)
+    def _TIMESTAMP_LTZ_to_python(self, value, ctx):
+        t, _ = self._pre_TIMESTAMP_LTZ_to_python(value, ctx)
         return t
 
-    def _TIMESTAMP_LTZ_numpy_to_python(self, value, col_desc, *_):
+    def _TIMESTAMP_LTZ_numpy_to_python(self, value, ctx):
         t, fraction_of_nanoseconds = self._pre_TIMESTAMP_LTZ_to_python(
-            value, col_desc)
+            value, ctx)
         ts = int(time.mktime(t.timetuple())) * 1000000000 + int(
             fraction_of_nanoseconds)
         return numpy.datetime64(ts, 'ns')
 
     _TIMESTAMP_to_python = _TIMESTAMP_LTZ_to_python
 
-    def _pre_TIMESTAMP_NTZ_to_python(self, value, col_desc):
+    def _pre_TIMESTAMP_NTZ_to_python(self, value, ctx):
         """TIMESTAMP NTZ to datetime
 
         No timezone info is attached.
         """
         tzoffset, microseconds, fraction_of_nanoseconds, nanoseconds = \
-            self._extract_timestamp(value, col_desc)
+            self._extract_timestamp(value, ctx)
 
         if tzoffset is None:
             return None, None, None
 
         return nanoseconds, microseconds, fraction_of_nanoseconds
 
-    def _TIMESTAMP_NTZ_to_python(self, value, col_desc, *_):
+    def _TIMESTAMP_NTZ_to_python(self, value, ctx):
         """
         TIMESTAMP NTZ to datetime
 
         No timezone info is attached.
         """
-        _, microseconds, _ = self._pre_TIMESTAMP_NTZ_to_python(value, col_desc)
+        _, microseconds, _ = self._pre_TIMESTAMP_NTZ_to_python(value, ctx)
         if microseconds is None:
             return None
 
@@ -292,23 +295,23 @@ def _TIMESTAMP_NTZ_to_python(self, value, col_desc, *_):
         t = ZERO_EPOCH + timedelta(seconds=(microseconds / float(1000000)))
         return t
 
-    def _TIMESTAMP_NTZ_numpy_to_python(self, value, col_desc, *_):
+    def _TIMESTAMP_NTZ_numpy_to_python(self, value, ctx):
         """
         TIMESTAMP NTZ to datetime64
 
         No timezone info is attached.
         """
-        nanoseconds, _, _ = self._pre_TIMESTAMP_NTZ_to_python(value, col_desc)
+        nanoseconds, _, _ = self._pre_TIMESTAMP_NTZ_to_python(value, ctx)
         return numpy.datetime64(nanoseconds, 'ns')
 
-    def _extract_time(self, value, col_desc):
+    def _extract_time(self, value, ctx):
         u"""Extracts time from raw data
 
         Returns a pair containing microseconds since midnight and nanoseconds
         since the last whole-numebr second. The last 6 digits of microseconds
         will be the same as the first 6 digits of nanoseconds.
         """
-        scale = col_desc[5]
+        scale = ctx['scale']
         try:
             value1 = decimal.Decimal(value)
             big_int = int(value1.scaleb(scale))  # removed fraction
@@ -321,13 +324,13 @@ def _extract_time(self, value, col_desc):
         except decimal.InvalidOperation:
             return None, None
 
-    def _TIME_to_python(self, value, col_desc, *_):
+    def _TIME_to_python(self, value, ctx):
         """
         TIME to formatted string, SnowflakeDateTime, or datetime.time
 
         No timezone is attached.
         """
-        microseconds, _ = self._extract_time(value, col_desc)
+        microseconds, _ = self._extract_time(value, ctx)
         ts = ZERO_EPOCH + timedelta(seconds=(microseconds / float(1000000)))
         return ts.time()
 
@@ -338,13 +341,13 @@ def _VARIANT_to_python(self, value, *_):
 
     _VARIANT_numpy_to_python = _VARIANT_to_python
 
-    def _OBJECT_to_python(self, value, col_desc, *_):
-        return self._VARIANT_to_python(value, col_desc)
+    def _OBJECT_to_python(self, value, *_):
+        return self._VARIANT_to_python(value)
 
     _OBJECT_numpy_to_python = _OBJECT_to_python
 
-    def _ARRAY_to_python(self, value, col_desc, *_):
-        return self._VARIANT_to_python(value, col_desc)
+    def _ARRAY_to_python(self, value, *_):
+        return self._VARIANT_to_python(value)
 
     _ARRAY_numpy_to_python = _ARRAY_to_python
 

converter_snowsql.py

@@ -56,6 +56,9 @@ def _get_format(self, type_name):
     # FROM Snowflake to Python objects
     #
     def to_python_method(self, type_name, row_type):
+        ctx = {
+            u'scale': row_type['scale'],
+        }
         try:
             fmt = None
             if is_timestamp_type_name(type_name):
@@ -64,8 +67,9 @@ def to_python_method(self, type_name, row_type):
                     datetime_class=SnowflakeDateTime)
             elif type_name == u'BINARY':
                 fmt = SnowflakeBinaryFormat(self._get_format(type_name))
+            ctx['fmt'] = fmt
             return getattr(self, u'_{type_name}_to_python'.format(
-                type_name=type_name)), fmt
+                type_name=type_name)), ctx
         except KeyError:
             # no type is defined, pass through it
             return self._TEXT_to_python, None
@@ -88,18 +92,19 @@ def _REAL_to_python(self, value, *_):
         """
         return value
 
-    def _BINARY_to_python(self, value, _, fmt):
+    def _BINARY_to_python(self, value, ctx):
         """
         BINARY to a string formatted by BINARY_OUTPUT_FORMAT
         """
-        return fmt.format(binary_to_python(value))
+        return ctx['fmt'].format(binary_to_python(value))
 
-    def _DATE_to_python(self, value, _, fmt):
+    def _DATE_to_python(self, value, ctx):
         """
         DATE to datetime
 
         No timezone is attached.
         """
+        fmt = ctx['fmt']
         try:
             t = ZERO_EPOCH + timedelta(seconds=int(value) * (24 * 60 * 60))
             if fmt:
@@ -118,29 +123,29 @@ def _DATE_to_python(self, value, _, fmt):
             return u'{year:d}-{month:02d}-{day:02d}'.format(
                 year=t.tm_year, month=t.tm_mon, day=t.tm_mday)
 
-    def _TIMESTAMP_TZ_to_python(self, value, col_desc, fmt):
+    def _TIMESTAMP_TZ_to_python(self, value, ctx):
         """
         TIMESTAMP TZ to datetime
 
         The timezone offset is piggybacked.
         """
         t, fraction_of_nanoseconds = self._pre_TIMESTAMP_TZ_to_python(
-            value, col_desc)
-        return _format_sftimestamp(fmt, t, fraction_of_nanoseconds)
+            value, ctx)
+        return _format_sftimestamp(ctx['fmt'], t, fraction_of_nanoseconds)
 
-    def _TIMESTAMP_LTZ_to_python(self, value, col_desc, fmt):
+    def _TIMESTAMP_LTZ_to_python(self, value, ctx):
         t, fraction_of_nanoseconds = self._pre_TIMESTAMP_LTZ_to_python(
-            value, col_desc)
-        return _format_sftimestamp(fmt, t, fraction_of_nanoseconds)
+            value, ctx)
+        return _format_sftimestamp(ctx['fmt'], t, fraction_of_nanoseconds)
 
-    def _TIMESTAMP_NTZ_to_python(self, value, col_desc, fmt):
+    def _TIMESTAMP_NTZ_to_python(self, value, ctx):
         """
         TIMESTAMP NTZ to Snowflake Formatted String
 
         No timezone info is attached.
         """
         _, microseconds, fraction_of_nanoseconds = \
-            self._pre_TIMESTAMP_NTZ_to_python(value, col_desc)
+            self._pre_TIMESTAMP_NTZ_to_python(value, ctx)
         if microseconds is None:
             return None
         try:
@@ -151,16 +156,16 @@ def _TIMESTAMP_NTZ_to_python(self, value, col_desc, fmt):
                 "ms). Falling back to use struct_time.",
                 microseconds)
             t = time.gmtime(microseconds / float(1000000))
-        return _format_sftimestamp(fmt, t, fraction_of_nanoseconds)
+        return _format_sftimestamp(ctx['fmt'], t, fraction_of_nanoseconds)
 
-    def _TIME_to_python(self, value, col_desc, fmt):
+    def _TIME_to_python(self, value, ctx):
         """
         TIME to formatted string, SnowflakeDateTime, or datetime.time
 
         No timezone is attached.
         """
         microseconds, fraction_of_nanoseconds = \
-            self._extract_time(value, col_desc)
+            self._extract_time(value, ctx)
 
         try:
             t = ZERO_EPOCH + timedelta(seconds=(microseconds / float(1000000)))
@@ -170,4 +175,4 @@ def _TIME_to_python(self, value, col_desc, fmt):
                 "ms). Falling back to use struct_time.",
                 microseconds)
             t = time.gmtime(microseconds / float(1000000))
-        return _format_sftimestamp(fmt, t, fraction_of_nanoseconds)
+        return _format_sftimestamp(ctx['fmt'], t, fraction_of_nanoseconds)