Merge pull request #98 from alimanfoo/msgpack-alimanfoo-20181106

Fix msgpack issues and warnings

Author: alimanfoo

fixture/msgpack2/array.08.npy

@@ -1,4 +1,6 @@
 {
-    "encoding": "utf-8",
-    "id": "msgpack2"
+    "id": "msgpack2",
+    "raw": false,
+    "use_bin_type": true,
+    "use_single_float": false
 }

fixture/msgpack2/array.09.npy

@@ -0,0 +1 @@
+���foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�|O���

fixture/msgpack2/codec.00/config.json

@@ -18,6 +18,17 @@ class MsgPack(Codec):
         The encoding format has been changed to include the array shape in the encoded
         data, which ensures that all object arrays can be correctly encoded and decoded.
 
+    Parameters
+    ----------
+    use_single_float : bool, optional
+        Use single precision float type for float.
+    use_bin_type : bool, optional
+        Use bin type introduced in msgpack spec 2.0 for bytes. It also enables str8 type
+        for unicode.
+    raw : bool, optional
+        If true, unpack msgpack raw to Python bytes. Otherwise, unpack to Python str
+        (or unicode on Python 2) by decoding with UTF-8 encoding.
+
     Examples
     --------
     >>> import numcodecs
@@ -39,19 +50,22 @@ class MsgPack(Codec):
 
     codec_id = 'msgpack2'
 
-    def __init__(self, encoding='utf-8'):
-        self.encoding = encoding
+    def __init__(self, use_single_float=False, use_bin_type=True, raw=False):
+        self.use_single_float = use_single_float
+        self.use_bin_type = use_bin_type
+        self.raw = raw
 
     def encode(self, buf):
         buf = np.asanyarray(buf)
         items = buf.tolist()
         items.append(buf.dtype.str)
         items.append(buf.shape)
-        return msgpack.packb(items, encoding=self.encoding)
+        return msgpack.packb(items, use_bin_type=self.use_bin_type,
+                             use_single_float=self.use_single_float)
 
     def decode(self, buf, out=None):
         buf = buffer_tobytes(buf)
-        items = msgpack.unpackb(buf, encoding=self.encoding)
+        items = msgpack.unpackb(buf, raw=self.raw)
         dec = np.empty(items[-1], dtype=items[-2])
         dec[:] = items[:-2]
         if out is not None:
@@ -62,13 +76,18 @@ def decode(self, buf, out=None):
 
     def get_config(self):
         return dict(id=self.codec_id,
-                    encoding=self.encoding)
+                    raw=self.raw,
+                    use_single_float=self.use_single_float,
+                    use_bin_type=self.use_bin_type)
 
     def __repr__(self):
-        return 'MsgPack(encoding=%r)' % self.encoding
+        return (
+            'MsgPack(raw={!r}, use_bin_type={!r}, use_single_float={!r})'
+            .format(self.raw, self.use_bin_type, self.use_single_float)
+        )
 
 
-class LegacyMsgPack(MsgPack):
+class LegacyMsgPack(Codec):
     """Deprecated MsgPack codec.
 
     .. deprecated:: 0.6.0
@@ -82,6 +101,9 @@ class LegacyMsgPack(MsgPack):
 
     codec_id = 'msgpack'
 
+    def __init__(self, encoding='utf-8'):
+        self.encoding = encoding
+
     def encode(self, buf):
         buf = np.asanyarray(buf)
         items = buf.tolist()
@@ -98,5 +120,9 @@ def decode(self, buf, out=None):
         else:
             return dec
 
+    def get_config(self):
+        return dict(id=self.codec_id,
+                    encoding=self.encoding)
+
     def __repr__(self):
         return 'LegacyMsgPack(encoding=%r)' % self.encoding

fixture/msgpack2/codec.00/encoded.06.dat

@@ -1,59 +1,78 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, division
 import unittest
-import itertools
+import warnings
 
 
 import numpy as np
 
 
 try:
     from numcodecs.msgpacks import LegacyMsgPack, MsgPack
-    codecs = [LegacyMsgPack(), MsgPack()]
+    default_codec = MsgPack()
+    # N.B., legacy codec is broken, see tests below. Also legacy code generates
+    # PendingDeprecationWarning due to use of encoding argument, which we ignore here
+    # as not relevant.
+    legacy_codec = LegacyMsgPack()
 except ImportError:  # pragma: no cover
     raise unittest.SkipTest("msgpack not available")
 
 
 from numcodecs.tests.common import (check_config, check_repr, check_encode_decode_array,
                                     check_backwards_compatibility, greetings)
+from numcodecs.compat import text_type, binary_type, PY2
 
 
 # object array with strings
 # object array with mix strings / nans
 # object array with mix of string, int, float
 # ...
 arrays = [
-    np.array(['foo', 'bar', 'baz'] * 300, dtype=object),
-    np.array([['foo', 'bar', np.nan]] * 300, dtype=object),
-    np.array(['foo', 1.0, 2] * 300, dtype=object),
+    np.array([u'foo', u'bar', u'baz'] * 300, dtype=object),
+    np.array([[u'foo', u'bar', np.nan]] * 300, dtype=object),
+    np.array([u'foo', 1.0, 2] * 300, dtype=object),
     np.arange(1000, dtype='i4'),
-    np.array(['foo', 'bar', 'baz'] * 300),
-    np.array(['foo', ['bar', 1.0, 2], {'a': 'b', 'c': 42}] * 300, dtype=object),
+    np.array([u'foo', u'bar', u'baz'] * 300),
+    np.array([u'foo', [u'bar', 1.0, 2], {u'a': u'b', u'c': 42}] * 300, dtype=object),
     np.array(greetings * 100),
     np.array(greetings * 100, dtype=object),
+    np.array([b'foo', b'bar', b'baz'] * 300, dtype=object),
+    np.array([g.encode('utf-8') for g in greetings] * 100, dtype=object),
 ]
 
 
+legacy_arrays = arrays[:8]
+
+
 def test_encode_decode():
-    for arr, codec in itertools.product(arrays, codecs):
-        check_encode_decode_array(arr, codec)
+
+    for arr in arrays:
+        check_encode_decode_array(arr, default_codec)
+
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', PendingDeprecationWarning)
+        for arr in legacy_arrays:
+            check_encode_decode_array(arr, legacy_codec)
 
 
 def test_config():
-    for codec in codecs:
+    for codec in [default_codec, legacy_codec]:
         check_config(codec)
 
 
 def test_repr():
-    check_repr("MsgPack(encoding='utf-8')")
-    check_repr("MsgPack(encoding='ascii')")
+    check_repr("MsgPack(raw=False, use_bin_type=True, use_single_float=False)")
+    check_repr("MsgPack(raw=True, use_bin_type=False, use_single_float=True)")
     check_repr("LegacyMsgPack(encoding='utf-8')")
     check_repr("LegacyMsgPack(encoding='ascii')")
 
 
 def test_backwards_compatibility():
-    for codec in codecs:
-        check_backwards_compatibility(codec.codec_id, arrays, [codec])
+    check_backwards_compatibility(default_codec.codec_id, arrays, [default_codec])
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', PendingDeprecationWarning)
+        check_backwards_compatibility(legacy_codec.codec_id, legacy_arrays,
+                                      [legacy_codec])
 
 
 def test_non_numpy_inputs():
@@ -64,16 +83,21 @@ def test_non_numpy_inputs():
         [[0, 1], [2, 3]],
         [[0], [1], [2, 3]],
         [[[0, 0]], [[1, 1]], [[2, 3]]],
-        ["1"],
-        ["11", "11"],
-        ["11", "1", "1"],
+        [u"1"],
+        [u"11", u"11"],
+        [u"11", u"1", u"1"],
         [{}],
-        [{"key": "value"}, ["list", "of", "strings"]],
+        [{u"key": u"value"}, [u"list", u"of", u"strings"]],
+        [b"1"],
+        [b"11", b"11"],
+        [b"11", b"1", b"1"],
+        [{b"key": b"value"}, [b"list", b"of", b"strings"]],
     ]
     for input_data in data:
-        for codec in codecs:
-            output_data = codec.decode(codec.encode(input_data))
-            assert np.array_equal(np.array(input_data), output_data)
+        actual = default_codec.decode(default_codec.encode(input_data))
+        expect = np.array(input_data)
+        assert expect.shape == actual.shape
+        assert np.array_equal(expect, actual)
 
 
 def test_legacy_codec_broken():
@@ -85,7 +109,9 @@ def test_legacy_codec_broken():
     a[0] = [0, 1]
     a[1] = [2, 3]
     codec = LegacyMsgPack()
-    b = codec.decode(codec.encode(a))
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', PendingDeprecationWarning)
+        b = codec.decode(codec.encode(a))
     assert a.shape == (2,)
     assert b.shape == (2, 2)
     assert not np.array_equal(a, b)
@@ -94,3 +120,66 @@ def test_legacy_codec_broken():
     codec = MsgPack()
     b = codec.decode(codec.encode(a))
     assert np.array_equal(a, b)
+    assert a.shape == b.shape
+
+
+def test_encode_decode_shape_dtype_preserved():
+    for arr in arrays:
+        actual = default_codec.decode(default_codec.encode(arr))
+        assert arr.shape == actual.shape
+        assert arr.dtype == actual.dtype
+
+
+def test_bytes():
+    # test msgpack behaviour with bytes and str (unicode)
+    bytes_arr = np.array([b'foo', b'bar', b'baz'], dtype=object)
+    unicode_arr = np.array([u'foo', u'bar', u'baz'], dtype=object)
+
+    # raw=False (default)
+    codec = MsgPack()
+    # works for bytes array, round-trips bytes to bytes
+    b = codec.decode(codec.encode(bytes_arr))
+    assert np.array_equal(bytes_arr, b)
+    assert isinstance(b[0], binary_type)
+    assert b[0] == b'foo'
+    # works for unicode array, round-trips unicode to unicode
+    b = codec.decode(codec.encode(unicode_arr))
+    assert np.array_equal(unicode_arr, b)
+    assert isinstance(b[0], text_type)
+    assert b[0] == u'foo'
+
+    # raw=True
+    codec = MsgPack(raw=True)
+    # works for bytes array, round-trips bytes to bytes
+    b = codec.decode(codec.encode(bytes_arr))
+    assert np.array_equal(bytes_arr, b)
+    assert isinstance(b[0], binary_type)
+    assert b[0] == b'foo'
+    # broken for unicode array, round-trips unicode to bytes
+    b = codec.decode(codec.encode(unicode_arr))
+    if PY2:
+        # PY2 considers b'foo' and u'foo' to be equal
+        assert np.array_equal(unicode_arr, b)
+    else:
+        assert not np.array_equal(unicode_arr, b)
+    assert isinstance(b[0], binary_type)
+    assert b[0] == b'foo'
+
+    # legacy codec
+    codec = LegacyMsgPack()
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', PendingDeprecationWarning)
+        # broken for bytes array, round-trips bytes to unicode
+        b = codec.decode(codec.encode(bytes_arr))
+        if PY2:
+            # PY2 considers b'foo' and u'foo' to be equal
+            assert np.array_equal(unicode_arr, b)
+        else:
+            assert not np.array_equal(bytes_arr, b)
+        assert isinstance(b[0], text_type)
+        assert b[0] == u'foo'
+        # works for unicode array, round-trips unicode to unicode
+        b = codec.decode(codec.encode(unicode_arr))
+        assert np.array_equal(unicode_arr, b)
+        assert isinstance(b[0], text_type)
+        assert b[0] == u'foo'