Merge 'zarr-developers/master' into 'funkey/master'

Author: jakirkham

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,7 +2,7 @@
 
 TODO:
 * [ ] Unit tests and/or doctests in docstrings
-* [ ] ``tox -e py36`` passes locally
+* [ ] ``tox -e py37`` passes locally
 * [ ] ``tox -e py27`` passes locally
 * [ ] Docstrings and API docs for any new/modified user-facing classes and functions
 * [ ] Changes documented in docs/release.rst

.travis.yml

@@ -11,10 +11,14 @@ addons:
     packages:
       - liblzma-dev
 
-python:
-  - 2.7
-  - 3.5
-  - 3.6
+matrix:
+  include:
+    - python: 2.7
+    - python: 3.5
+    - python: 3.6
+    - python: 3.7
+      dist: xenial
+      sudo: true
 
 install:
   - pip install -U tox-travis coveralls pip setuptools wheel

appveyor.yml

@@ -31,6 +31,12 @@ environment:
     - PYTHON: "C:\\Python36-x64"
       PYTHON_VERSION: "3.6"
 
+    - PYTHON: "C:\\Python37"
+      PYTHON_VERSION: "3.7"
+
+    - PYTHON: "C:\\Python37-x64"
+      PYTHON_VERSION: "3.7"
+
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
   - git submodule update --init --recursive

docs/contributing.rst

@@ -90,7 +90,7 @@ you have cloned the NumCodecs source code and your current working directory is
 the repository, you can do something like the following::
 
     $ mkdir -p ~/pyenv/numcodecs-dev
-    $ virtualenv --no-site-packages --python=/usr/bin/python3.6 ~/pyenv/numcodecs-dev
+    $ virtualenv --no-site-packages --python=/usr/bin/python3.7 ~/pyenv/numcodecs-dev
     $ source ~/pyenv/numcodecs-dev/bin/activate
     $ pip install -r requirements_dev.txt
     $ python setup.py build_ext --inplace
@@ -145,11 +145,11 @@ To also run the doctests within docstrings, run::
 Tests can be run under different Python versions using tox. E.g. (assuming you have the
 corresponding Python interpreters installed on your system)::
 
-    $ tox -e py27,py35,py36
+    $ tox -e py27,py35,py36,py37
 
-NumCodecs currently supports Python 2.7 and Python 3.5-3.6, so the above command must
-succeed before code can be accepted into the main code base. Note that only the py36
-tox environment runs the doctests, i.e., doctests only need to succeed under Python 3.6.
+NumCodecs currently supports Python 2.7 and Python 3.5-3.7, so the above command must
+succeed before code can be accepted into the main code base. Note that only the py37
+tox environment runs the doctests, i.e., doctests only need to succeed under Python 3.7.
 
 All tests are automatically run via Travis (Linux) and AppVeyor (Windows) continuous
 integration services for every pull request. Tests must pass under both services before
@@ -164,14 +164,14 @@ Conformance can be checked by running::
 
     $ flake8 --max-line-length=100 numcodecs
 
-This is automatically run when invoking ``tox -e py36``.
+This is automatically run when invoking ``tox -e py37``.
 
 Test coverage
 ~~~~~~~~~~~~~
 
 NumCodecs maintains 100% test coverage under the latest Python stable release (currently
-Python 3.6). Both unit tests and docstring doctests are included when computing
-coverage. Running ``tox -e py36`` will automatically run the test suite with coverage
+Python 3.7). Both unit tests and docstring doctests are included when computing
+coverage. Running ``tox -e py37`` will automatically run the test suite with coverage
 and produce a coverage report. This should be 100% before code can be accepted into the
 main code base.
 
@@ -185,7 +185,7 @@ Documentation
 Docstrings for user-facing classes and functions should follow the `numpydoc
 <https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt>`_ standard,
 including sections for Parameters and Examples. All examples will be run as doctests
-under Python 3.6.
+under Python 3.7.
 
 NumCodecs uses Sphinx for documentation, hosted on readthedocs.org. Documentation is
 written in the RestructuredText markup language (.rst files) in the ``docs`` folder.

docs/release.rst

@@ -18,8 +18,14 @@ Release notes
 * Updated the msgpack dependency (by :user:`Jerome Kelleher <jeromekelleher>`;
   :issue:`74`, :issue:`75`).
 
+* Allow :class:`Blosc` compressor to run on systems where locks are not present (by
+  :user:`Marcus Kinsella <mckinsel>`, :issue:`83`; and :user:`Tom White <tomwhite>`,
+  :issue:`93`).
+
 * Drop Python 3.4 (by :user:`John Kirkham <jakirkham>`; :issue:`89`).
 
+* Add Python 3.7 (by :user:`John Kirkham <jakirkham>`; :issue:`92`).
+
 * Add codec :class:`numcodecs.gzip.GZip` to replace ``gzip`` alias for ``zlib``,
   which was incorrect (by :user:`Jan Funke <funkey>`; :issue:`87`).
 
@@ -58,7 +64,7 @@ Release notes
 
 * Add support for encoding None values in VLen... codecs (:issue:`59`).
 
- 
+
 .. _release_0.5.1:
 
 0.5.1

numcodecs/__init__.py

@@ -46,7 +46,10 @@
     from numcodecs.blosc import Blosc
     register_codec(Blosc)
     # initialize blosc
-    ncores = multiprocessing.cpu_count()
+    try:
+        ncores = multiprocessing.cpu_count()
+    except OSError:
+        ncores = 1
     blosc.init()
     blosc.set_nthreads(min(8, ncores))
     atexit.register(blosc.destroy)

numcodecs/blosc.pyx

@@ -76,8 +76,10 @@ AUTOSHUFFLE = -1
 AUTOBLOCKS = 0
 
 # synchronization
-mutex = multiprocessing.Lock()
-
+try:
+    mutex = multiprocessing.Lock()
+except OSError:
+    mutex = None
 
 # store ID of process that first loads the module, so we can detect a fork later
 _importer_pid = os.getpid()
@@ -405,6 +407,10 @@ def _get_use_threads():
     global use_threads
     proc = multiprocessing.current_process()
 
+    # check if locks are available, and if not no threads
+    if not mutex:
+        return False
+
     # check for fork
     if proc.pid != _importer_pid:
         # If this module has been imported in the parent process, and the current process

numcodecs/fixedscaleoffset.py

@@ -38,33 +38,33 @@ class FixedScaleOffset(Codec):
     >>> import numpy as np
     >>> x = np.linspace(1000, 1001, 10, dtype='f8')
     >>> x
-    array([ 1000.        ,  1000.11111111,  1000.22222222,  1000.33333333,
-            1000.44444444,  1000.55555556,  1000.66666667,  1000.77777778,
-            1000.88888889,  1001.        ])
+    array([1000.        , 1000.11111111, 1000.22222222, 1000.33333333,
+           1000.44444444, 1000.55555556, 1000.66666667, 1000.77777778,
+           1000.88888889, 1001.        ])
     >>> codec = numcodecs.FixedScaleOffset(offset=1000, scale=10, dtype='f8', astype='u1')
     >>> y1 = codec.encode(x)
     >>> y1
     array([ 0,  1,  2,  3,  4,  6,  7,  8,  9, 10], dtype=uint8)
     >>> z1 = codec.decode(y1)
     >>> z1
-    array([ 1000. ,  1000.1,  1000.2,  1000.3,  1000.4,  1000.6,  1000.7,
-            1000.8,  1000.9,  1001. ])
+    array([1000. , 1000.1, 1000.2, 1000.3, 1000.4, 1000.6, 1000.7,
+           1000.8, 1000.9, 1001. ])
     >>> codec = numcodecs.FixedScaleOffset(offset=1000, scale=10**2, dtype='f8', astype='u1')
     >>> y2 = codec.encode(x)
     >>> y2
-    array([  0,  11,  22,  33,  44,  56,  67,  78,  89, 100], dtype=uint8)
+    array([ 0,  11,  22,  33,  44,  56,  67,  78,  89, 100], dtype=uint8)
     >>> z2 = codec.decode(y2)
     >>> z2
-    array([ 1000.  ,  1000.11,  1000.22,  1000.33,  1000.44,  1000.56,
-            1000.67,  1000.78,  1000.89,  1001.  ])
+    array([1000.  , 1000.11, 1000.22, 1000.33, 1000.44, 1000.56,
+           1000.67, 1000.78, 1000.89, 1001.  ])
     >>> codec = numcodecs.FixedScaleOffset(offset=1000, scale=10**3, dtype='f8', astype='u2')
     >>> y3 = codec.encode(x)
     >>> y3
-    array([   0,  111,  222,  333,  444,  556,  667,  778,  889, 1000], dtype=uint16)
+    array([ 0,  111,  222,  333,  444,  556,  667,  778,  889, 1000], dtype=uint16)
     >>> z3 = codec.decode(y3)
     >>> z3
-    array([ 1000.   ,  1000.111,  1000.222,  1000.333,  1000.444,  1000.556,
-            1000.667,  1000.778,  1000.889,  1001.   ])
+    array([1000.   , 1000.111, 1000.222, 1000.333, 1000.444, 1000.556,
+           1000.667, 1000.778, 1000.889, 1001.   ])
 
     See Also
     --------

numcodecs/packbits.py

@@ -23,7 +23,7 @@ class PackBits(Codec):
     array([  4, 144], dtype=uint8)
     >>> z = codec.decode(y)
     >>> z
-    array([ True, False, False,  True], dtype=bool)
+    array([ True, False, False,  True])
 
     Notes
     -----

numcodecs/quantize.py

@@ -28,20 +28,20 @@ class Quantize(Codec):
     >>> import numpy as np
     >>> x = np.linspace(0, 1, 10, dtype='f8')
     >>> x
-    array([ 0.        ,  0.11111111,  0.22222222,  0.33333333,  0.44444444,
-            0.55555556,  0.66666667,  0.77777778,  0.88888889,  1.        ])
+    array([0.        , 0.11111111, 0.22222222, 0.33333333, 0.44444444,
+           0.55555556, 0.66666667, 0.77777778, 0.88888889, 1.        ])
     >>> codec = numcodecs.Quantize(digits=1, dtype='f8')
     >>> codec.encode(x)
-    array([ 0.    ,  0.125 ,  0.25  ,  0.3125,  0.4375,  0.5625,  0.6875,
-            0.75  ,  0.875 ,  1.    ])
+    array([0.    , 0.125 , 0.25  , 0.3125, 0.4375, 0.5625, 0.6875,
+           0.75  , 0.875 , 1.    ])
     >>> codec = numcodecs.Quantize(digits=2, dtype='f8')
     >>> codec.encode(x)
-    array([ 0.       ,  0.109375 ,  0.21875  ,  0.3359375,  0.4453125,
-            0.5546875,  0.6640625,  0.78125  ,  0.890625 ,  1.       ])
+    array([0.       , 0.109375 , 0.21875  , 0.3359375, 0.4453125,
+           0.5546875, 0.6640625, 0.78125  , 0.890625 , 1.       ])
     >>> codec = numcodecs.Quantize(digits=3, dtype='f8')
     >>> codec.encode(x)
-    array([ 0.        ,  0.11132812,  0.22265625,  0.33300781,  0.44433594,
-            0.55566406,  0.66699219,  0.77734375,  0.88867188,  1.        ])
+    array([0.        , 0.11132812, 0.22265625, 0.33300781, 0.44433594,
+           0.55566406, 0.66699219, 0.77734375, 0.88867188, 1.        ])
 
     See Also
     --------