Merge pull request #59 from jcrobak/codecs-fix

Fix the utf-8 converted type issue

Author: jcrobak

parquet/converted_types.py

@@ -73,7 +73,7 @@ def convert_column(data, schemae):
     elif ctype == parquet_thrift.ConvertedType.TIMESTAMP_MILLIS:
         return [datetime.datetime.utcfromtimestamp(d / 1000.0) for d in data]
     elif ctype == parquet_thrift.ConvertedType.UTF8:
-        return list(codecs.iterdecode(data, "utf-8"))
+        return [codecs.decode(item, "utf-8") for item in data]
     elif ctype == parquet_thrift.ConvertedType.UINT_8:
         return _convert_unsigned(data, 'b')
     elif ctype == parquet_thrift.ConvertedType.UINT_16:

test/test_converted_types.py

@@ -134,6 +134,19 @@ def test_utf8(self):
             'foo👾'
         )
 
+    def test_utf8_empty_string(self):
+        """Test bytes representing utf-8 string with empty strings."""
+        schema = pt.SchemaElement(
+            type=pt.Type.BYTE_ARRAY,
+            name="test",
+            converted_type=pt.ConvertedType.UTF8
+        )
+        data = [b'', b'foo\xf0\x9f\x91\xbe', b'']
+        self.assertEqual(
+            convert_column(data, schema),
+            ['', 'foo👾', '']
+        )
+
     def test_json(self):
         """Test bytes representing json."""
         schema = pt.SchemaElement(