docstrings and comments; unicode array.array disallowed

Author: alimanfoo

numcodecs/compat.py

@@ -34,17 +34,42 @@ def ensure_text(l, encoding='utf-8'):
 
 
 def ensure_ndarray(buf, dtype=None):
-    """TODO"""
+    """Convenience function to coerce `buf` to a numpy array, if it is not already a
+    numpy array.
 
-    # make that we create an array from a memory buffer with no copy
+    Parameters
+    ----------
+    buf : array-like or bytes-like
+        A numpy array or any object exporting a buffer interface.
+    dtype : dtype, optional
+        Request that the data be viewed as the given dtype.
 
-    if isinstance(buf, np.ndarray):
+    Returns
+    -------
+    arr : ndarray
+        A numpy array, sharing memory with `buf`.
+
+    Notes
+    -----
+    This function will not create a copy under any circumstances, it is guaranteed to
+    return a view on memory exported by `buf`.
 
+    """
+
+    if isinstance(buf, np.ndarray):
         # already a numpy array
         arr = buf
 
+    elif isinstance(buf, array.array) and buf.typecode == 'u':
+        # guard condition, do not support array.array with unicode type, this is
+        # problematic because numpy does not support it on all platforms
+        raise TypeError('array.array with unicode type is not supported')
+
     else:
 
+        # N.B., first take a memoryview to make sure that we subsequently create a
+        # numpy array from a memory buffer with no copy
+
         if PY2:  # pragma: py3 no cover
             try:
                 mem = memoryview(buf)
@@ -59,30 +84,45 @@ def ensure_ndarray(buf, dtype=None):
         arr = np.array(mem, copy=False)
 
         if PY2 and isinstance(buf, array.array):  # pragma: py3 no cover
-
             # type information will not have been propagated via the old-style buffer
-            # interface, so we have to manually hack it after the fact
-            if buf.typecode == 'u':
-                t = 'U1'
-            else:
-                t = buf.typecode
-            arr = arr.view(t)
+            # interface, so we have to manually hack it back in after the fact
+            arr = arr.view(buf.typecode)
 
     if dtype is not None:
-
         # view as requested dtype
         arr = arr.view(dtype)
 
     return arr
 
 
 def ensure_contiguous_ndarray(buf, dtype=None):
-    """TODO"""
+    """Convenience function to coerce `buf` to a numpy array, if it is not already a
+    numpy array. Also ensures that the returned value exports fully contiguous memory,
+    and supports the new-style buffer interface.
+
+    Parameters
+    ----------
+    buf : array-like or bytes-like
+        A numpy array or any object exporting a buffer interface.
+    dtype : dtype, optional
+        Request that the data be viewed as the given dtype.
+
+    Returns
+    -------
+    arr : ndarray
+        A numpy array, sharing memory with `buf`.
+
+    Notes
+    -----
+    This function will not create a copy under any circumstances, it is guaranteed to
+    return a view on memory exported by `buf`.
+
+    """
 
     # ensure input is a numpy array
     arr = ensure_ndarray(buf, dtype=dtype)
 
-    # check for datetime or timedelta ndarray, cannot take a memoryview of those
+    # check for datetime or timedelta ndarray, the buffer interface doesn't support those
     if isinstance(buf, np.ndarray) and buf.dtype.kind in 'Mm':
         arr = arr.view(np.int64)
 
@@ -93,7 +133,6 @@ def ensure_contiguous_ndarray(buf, dtype=None):
 
     # check memory is contiguous, if so flatten
     if arr.flags.c_contiguous or arr.flags.f_contiguous:
-
         # can flatten without copy
         arr = arr.reshape(-1, order='A')
 
@@ -103,18 +142,18 @@ def ensure_contiguous_ndarray(buf, dtype=None):
     return arr
 
 
-def ensure_bytes(o):
-    """Obtain a bytes object from memory exposed by `o`."""
+def ensure_bytes(buf):
+    """Obtain a bytes object from memory exposed by `buf`."""
 
-    if not isinstance(o, binary_type):
+    if not isinstance(buf, binary_type):
 
         # go via numpy, for convenience
-        a = ensure_contiguous_ndarray(o)
+        arr = ensure_ndarray(buf)
 
         # create bytes
-        o = a.tobytes()
+        buf = arr.tobytes(order='A')
 
-    return o
+    return buf
 
 
 def ndarray_copy(src, dst):

numcodecs/tests/test_compat.py

@@ -33,14 +33,22 @@ def test_ensure_contiguous_ndarray_shares_memory():
         ('u', 4, array.array('I', b'qwertyuiqwertyui')),
         ('f', 4, array.array('f', b'qwertyuiqwertyui')),
         ('f', 8, array.array('d', b'qwertyuiqwertyui')),
-        ('U', 4, array.array('u', u'qwertyuiqwertyui')),
+        ('i', 1, array.array('b', b'qwertyuiqwertyui')),
+        ('u', 1, array.array('B', b'qwertyuiqwertyui')),
         ('u', 1, mmap.mmap(-1, 10))
     ]
-    for typ, siz, buf in typed_bufs:
+    if PY2:  # pragma: py3 no cover
+        typed_bufs.append(
+            ('S', 1, array.array('c', b'qwertyuiqwertyui')),
+        )
+    for expected_kind, expected_itemsize, buf in typed_bufs:
         a = ensure_contiguous_ndarray(buf)
         assert isinstance(a, np.ndarray)
-        assert typ == a.dtype.kind
-        assert siz == a.dtype.itemsize
+        assert expected_kind == a.dtype.kind
+        if isinstance(buf, array.array):
+            assert buf.itemsize == a.dtype.itemsize
+        else:
+            assert expected_itemsize == a.dtype.itemsize
         if PY2:  # pragma: py3 no cover
             assert np.shares_memory(a, np.getbuffer(buf))
         else:  # pragma: py2 no cover
@@ -59,8 +67,13 @@ def test_ensure_contiguous_ndarray_invalid_inputs():
     with pytest.raises(ValueError):
         ensure_contiguous_ndarray(np.arange(100)[::2])
 
+    # unicode array.array not allowed
+    a = array.array('u', u'qwertyuiqwertyui')
+    with pytest.raises(TypeError):
+        ensure_contiguous_ndarray(a)
 
-def test_ensure_contiguous_ndarray_writable():
+
+def test_ensure_contiguous_ndarray_writeable():
     # check that the writeability of the underlying buffer is preserved
     for writeable in [False, True]:
         a = np.arange(100)