review docs; sure-up blosc refactor

alimanfoo · alimanfoo · commit d8b842f2b81d · 2016-09-19T11:57:34.000+01:00
diff --git a/README.rst b/README.rst
@@ -1,7 +1,8 @@
-numcodecs
+Numcodecs
 =========
 
-TODO
+Numcodecs is a Python package providing buffer compression and transformation 
+codecs for use in data storage and communication applications.
 
 .. image:: https://travis-ci.org/alimanfoo/numcodecs.svg?branch=master
     :target: https://travis-ci.org/alimanfoo/numcodecs
diff --git a/adhoc/blosc_memleak_check.py b/adhoc/blosc_memleak_check.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, print_function, division
+import sys
+
+
+import numcodecs as codecs
+from numcodecs import blosc
+import numpy as np
+from numpy.testing import assert_array_equal
+
+
+codec = codecs.Blosc()
+data = np.arange(int(sys.argv[1]))
+for i in range(int(sys.argv[2])):
+    enc = codec.encode(data)
+    dec = codec.decode(enc)
+    arr = np.frombuffer(dec, dtype=data.dtype)
+    assert_array_equal(data, arr)
diff --git a/docs/abc.rst b/docs/abc.rst
@@ -1,9 +1,10 @@
-Codec
-=====
-.. module:: numcodecs.abc
+Codec API
+=========
+.. automodule:: numcodecs.abc
 
 .. autoclass:: Codec
 
+    .. autoattribute:: codec_id
     .. automethod:: encode
     .. automethod:: decode
     .. automethod:: get_config
diff --git a/docs/blosc.rst b/docs/blosc.rst
@@ -1,6 +1,6 @@
 Blosc
 =====
-.. module:: numcodecs.blosc
+.. automodule:: numcodecs.blosc
 
 .. autoclass:: Blosc
 
diff --git a/docs/bz2.rst b/docs/bz2.rst
@@ -1,5 +1,5 @@
 BZ2
 ===
-.. module:: numcodecs.bz2
+.. automodule:: numcodecs.bz2
 
 .. autoclass:: BZ2
diff --git a/docs/categorize.rst b/docs/categorize.rst
@@ -1,5 +1,5 @@
 Categorize
 ==========
-.. module:: numcodecs.categorize
+.. automodule:: numcodecs.categorize
 
 .. autoclass:: Categorize
diff --git a/docs/checksum32.rst b/docs/checksum32.rst
@@ -1,6 +1,11 @@
 32-bit checksums
 ================
-.. module:: numcodecs.checksum32
+.. automodule:: numcodecs.checksum32
 
+CRC32
+-----
 .. autoclass:: CRC32
+
+Adler32
+-------
 .. autoclass:: Adler32
diff --git a/docs/delta.rst b/docs/delta.rst
@@ -1,5 +1,5 @@
 Delta
 =====
-.. module:: numcodecs.delta
+.. automodule:: numcodecs.delta
 
 .. autoclass:: Delta
diff --git a/docs/fixedscaleoffset.rst b/docs/fixedscaleoffset.rst
@@ -1,5 +1,5 @@
 FixedScaleOffset
 ================
-.. module:: numcodecs.fixedscaleoffset
+.. automodule:: numcodecs.fixedscaleoffset
 
 .. autoclass:: FixedScaleOffset
diff --git a/docs/index.rst b/docs/index.rst
@@ -1,15 +1,11 @@
 .. numcodecs documentation master file, created by
    sphinx-quickstart on Mon May  2 21:40:09 2016.
 
+
 Numcodecs
 =========
 
-This package contains compression and other codecs intended primarily for use
-with numerical data.
-
-If you have a question, find a bug, would like to make a suggestion or
-contribute code, please `raise an issue on GitHub
-<https://github.com/alimanfoo/numcodecs/issues>`_.
+.. automodule:: numcodecs
 
 Installation
 ------------
@@ -62,6 +58,7 @@ Contents
     packbits
     categorize
     checksum32
+    registry
     release
 
 Acknowledgments
diff --git a/docs/lzma.rst b/docs/lzma.rst
@@ -1,5 +1,5 @@
 LZMA
 ====
-.. module:: numcodecs.lzma
+.. automodule:: numcodecs.lzma
 
 .. autoclass:: LZMA
diff --git a/docs/packbits.rst b/docs/packbits.rst
@@ -1,5 +1,5 @@
 PackBits
 ========
-.. module:: numcodecs.packbits
+.. automodule:: numcodecs.packbits
 
 .. autoclass:: PackBits
diff --git a/docs/registry.rst b/docs/registry.rst
@@ -0,0 +1,6 @@
+Codec registry
+==============
+.. automodule:: numcodecs.registry
+
+.. autofunction:: get_codec
+.. autofunction:: register_codec
diff --git a/docs/release.rst b/docs/release.rst
@@ -6,4 +6,13 @@ Release notes
 0.0.0
 -----
 
-First release.
+First release. This version is a port of the ``codecs`` module from `Zarr
+<http://zarr.readthedocs.io>`_ 2.1.0. The following changes have been made from
+the original Zarr module:
+
+* Codec classes have been re-organized into separate modules, mostly one per
+  codec class, for ease of maintenance.
+* Two new codec classes have been added based on 32-bit checksums: CRC32 and
+  Adler32.
+* The Blosc extension has been refactored to remove code duplications related
+  to handling of buffer compatibility.
diff --git a/docs/zlib.rst b/docs/zlib.rst
@@ -1,5 +1,5 @@
 Zlib
 ====
-.. module:: numcodecs.zlib
+.. automodule:: numcodecs.zlib
 
 .. autoclass:: Zlib
diff --git a/numcodecs/__init__.py b/numcodecs/__init__.py
@@ -1,5 +1,23 @@
 # -*- coding: utf-8 -*-
 # flake8: noqa
+"""Numcodecs is a Python package providing buffer compression and
+transformation codecs for use in data storage and communication
+applications. These include:
+
+* Compression codecs, e.g., Zlib, BZ2, LZMA and Blosc.
+* Pre-compression filters, e.g., Delta, Quantize, FixedScaleOffset,
+  PackBits, Categorize.
+* Integrity checks, e.g., CRC32, Adler32.
+
+All codecs implement the same API, allowing codecs to be organized into
+pipelines in a variety of ways.
+
+If you have a question, find a bug, would like to make a suggestion or
+contribute code, please `raise an issue on GitHub
+<https://github.com/alimanfoo/numcodecs/issues>`_.
+
+"""
+
 from __future__ import absolute_import, print_function, division
 import multiprocessing
 import atexit
diff --git a/numcodecs/abc.py b/numcodecs/abc.py
@@ -1,4 +1,34 @@
 # -*- coding: utf-8 -*-
+"""This module defines the :class:`Codec` base class, a common interface for
+all codec classes.
+
+Codec classes must implement :func:`Codec.encode` and :func:`Codec.decode`
+methods. Inputs to and outputs from these methods may be any Python object
+exporting a contiguous buffer via the new-style Python protocol
+or :class:`array.array` under Python 2.
+
+Codec classes must implement a :func:`Codec.get_config` method,
+which must return a dictionary holding all configuration parameters
+required to enable encoding and decoding of data. The expectation is that
+these configuration parameters will be stored or communicated separately
+from encoded data, and thus the codecs do not need to store all encoding
+parameters within the encoded data. For broad compatibility,
+the configuration object must contain only JSON-serializable values. The
+configuration object must also contain an 'id' field storing the codec
+identifier (see below).
+
+Codec classes must implement a :func:`Codec.from_config` class method,
+which will return an instance of the class initiliazed from a configuration
+object.
+
+Finally, codec classes must set a `codec_id` class-level attribute. This
+must be a string. Two different codec classes may set the same value for the
+`codec_id` attribute if and only if they are fully compatible, meaning that
+(1) configuration parameters are the same, and (2) given the same
+configuration, one class could correctly decode data encoded by the
+other and vice versa.
+
+"""
 from __future__ import absolute_import, print_function, division
 
 
@@ -7,6 +37,7 @@ class Codec(object):  # pragma: no cover
 
     # override in sub-class
     codec_id = None
+    """Codec identifier."""
 
     def encode(self, buf):
         """Encode data in `buf`.
@@ -15,15 +46,13 @@ def encode(self, buf):
         ----------
         buf : buffer-like
             Data to be encoded. May be any object supporting the new-style
-            buffer protocol or `array.array` (only supports old-style buffer
-            protocol in PY2).
+            buffer protocol or `array.array` under Python 2.
 
         Returns
         -------
         enc : buffer-like
             Encoded data. May be any object supporting the new-style buffer
-            protocol or `array.array` (only supports old-style buffer
-            protocol in PY2).
+            protocol or `array.array` under Python 2.
 
         """
         # override in sub-class
@@ -36,40 +65,48 @@ def decode(self, buf, out=None):
         ----------
         buf : buffer-like
             Encoded data. May be any object supporting the new-style buffer
-            protocol or `array.array` (only supports old-style buffer
-            protocol in PY2).
+            protocol or `array.array` under Python 2.
         out : buffer-like, optional
             Writeable buffer to store decoded data.
 
         Returns
         -------
         dec : buffer-like
             Decoded data. May be any object supporting the new-style
-            buffer protocol or `array.array` (only supports old-style buffer
-            protocol in PY2).
+            buffer protocol or `array.array` under Python 2.
 
         """
         # override in sub-class
         raise NotImplementedError
 
     def get_config(self):
         """Return a dictionary holding configuration parameters for this
-        codec. Must include an 'id' field with the codec ID. All values must be
-        compatible with JSON encoding."""
+        codec. Must include an 'id' field with the codec identifier. All
+        values must be compatible with JSON encoding."""
+
         # override in sub-class if need special encoding of config values
+
+        # setup config object
         config = dict(id=self.codec_id)
-        # add in all non-private members
+
+        # by default, assume all non-private members are configuration
+        # parameters - override this in sub-class if not the case
         for k in self.__dict__:
             if not k.startswith('_'):
                 config[k] = getattr(self, k)
+
         return config
 
     @classmethod
     def from_config(cls, config):
         """Instantiate codec from a configuration object."""
         # N.B., assume at this point the 'id' field has been removed from
         # the config object
+
         # override in sub-class if need special decoding of config values
+
+        # by default, assume constructor accepts configuration parameters as
+        # keyword arguments without any special decoding
         return cls(**config)
 
     def __eq__(self, other):
@@ -80,9 +117,13 @@ def __eq__(self, other):
             return False
 
     def __repr__(self):
+
         # override in sub-class if need special representation
+
+        # by default, assume all non-private members are configuration
+        # parameters and valid keyword arguments to constructor function
+
         r = '%s(' % type(self).__name__
-        # by default, include all non-private members
         params = ['%s=%r' % (k, getattr(self, k))
                   for k in sorted(self.__dict__)
                   if not k.startswith('_')]
diff --git a/numcodecs/blosc.c b/numcodecs/blosc.c
diff --git a/numcodecs/blosc.pyx b/numcodecs/blosc.pyx
diff --git a/numcodecs/registry.py b/numcodecs/registry.py
