Use uniform chunking when specifying chunks as an integer (#456)

* Updates normalize_chunks behavior for integer chunks

* Return early for integer chunks case

* Updates hard coded hashes

* Adds item to release notes

* Updates docstrings

* Adds support for auto-chunking with -1

* Reference -1 in chunks in tutorial

* Updates release notes

* Updates s3fs.S3Map store type in tutorial

Author: jrbourbeau

docs/release.rst

@@ -4,6 +4,10 @@ Release notes
 Upcoming Release
 ----------------
 
+* Use uniform chunking for all dimensions when specifying ``chunks`` as an integer.
+  Also adds support for specifying ``-1`` to chunk across an entire dimension.
+  By :user:`James Bourbeau <jrbourbeau>`; :issue:`456`
+
 * Rename ``DictStore`` to ``MemoryStore``.
   By :user:`James Bourbeau <jrbourbeau>`; :issue:`455`
 

docs/tutorial.rst

@@ -791,7 +791,7 @@ Here is an example using S3Map to read an array created previously::
     Order              : C
     Read-only          : False
     Compressor         : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
-    Store type         : s3fs.mapping.S3Map
+    Store type         : fsspec.mapping.FSMap
     No. bytes          : 21
     Chunks initialized : 3/3
     >>> z[:]
@@ -1170,8 +1170,8 @@ better performance, at least when using the Blosc compression library.
 The optimal chunk shape will depend on how you want to access the data. E.g.,
 for a 2-dimensional array, if you only ever take slices along the first
 dimension, then chunk across the second dimenson. If you know you want to chunk
-across an entire dimension you can use ``None`` within the ``chunks`` argument,
-e.g.::
+across an entire dimension you can use ``None`` or ``-1`` within the ``chunks``
+argument, e.g.::
 
     >>> z1 = zarr.zeros((10000, 10000), chunks=(100, None), dtype='i4')
     >>> z1.chunks

zarr/creation.py

@@ -28,6 +28,8 @@ def create(shape, chunks=True, dtype=None, compressor='default',
     chunks : int or tuple of ints, optional
         Chunk shape. If True, will be guessed from `shape` and `dtype`. If
         False, will be set to `shape`, i.e., single chunk for the whole array.
+        If an int, the chunk size in each dimension will be given by the value
+        of `chunks`. Default is True.
     dtype : string or dtype, optional
         NumPy dtype.
     compressor : Codec, optional
@@ -369,6 +371,8 @@ def open_array(store=None, mode='a', shape=None, chunks=True, dtype=None,
     chunks : int or tuple of ints, optional
         Chunk shape. If True, will be guessed from `shape` and `dtype`. If
         False, will be set to `shape`, i.e., single chunk for the whole array.
+        If an int, the chunk size in each dimension will be given by the value
+        of `chunks`. Default is True.
     dtype : string or dtype, optional
         NumPy dtype.
     compressor : Codec, optional

zarr/tests/test_core.py

@@ -512,7 +512,7 @@ def test_hexdigest(self):
 
         # Check basic 2-D array
         z = self.create_array(shape=(20, 35,), chunks=10, dtype='<i4')
-        assert '4f797d7bdad0fa1c9fa8c80832efb891a68de104' == z.hexdigest()
+        assert 'c7190ad2bea1e9d2e73eaa2d3ca9187be1ead261' == z.hexdigest()
 
         # Check basic 1-D array with some data
         z = self.create_array(shape=(1050,), chunks=100, dtype='<i4')
@@ -1302,7 +1302,7 @@ def test_hexdigest(self):
 
         # Check basic 2-D array
         z = self.create_array(shape=(20, 35,), chunks=10, dtype='<i4')
-        assert 'dde44c72cc530bd6aae39b629eb15a2da627e5f9' == z.hexdigest()
+        assert '6c530b6b9d73e108cc5ee7b6be3d552cc994bdbe' == z.hexdigest()
 
         # Check basic 1-D array with some data
         z = self.create_array(shape=(1050,), chunks=100, dtype='<i4')
@@ -1357,7 +1357,7 @@ def test_hexdigest(self):
 
         # Check basic 2-D array
         z = self.create_array(shape=(20, 35,), chunks=10, dtype='<i4')
-        assert 'dde44c72cc530bd6aae39b629eb15a2da627e5f9' == z.hexdigest()
+        assert '6c530b6b9d73e108cc5ee7b6be3d552cc994bdbe' == z.hexdigest()
 
         # Check basic 1-D array with some data
         z = self.create_array(shape=(1050,), chunks=100, dtype='<i4')
@@ -1713,7 +1713,7 @@ def test_hexdigest(self):
 
         # Check basic 2-D array
         z = self.create_array(shape=(20, 35,), chunks=10, dtype='<i4')
-        assert '189690c5701d33a41cd7ce9aa0ac8dac49a69c51' == z.hexdigest()
+        assert 'ec2e008525ae09616dbc1d2408cbdb42532005c8' == z.hexdigest()
 
         # Check basic 1-D array with some data
         z = self.create_array(shape=(1050,), chunks=100, dtype='<i4')
@@ -1863,7 +1863,7 @@ def test_hexdigest(self):
 
         # Check basic 2-D array
         z = self.create_array(shape=(20, 35,), chunks=10, dtype='<i4')
-        assert 'de841ca276042993da53985de1e7769f5d0fc54d' == z.hexdigest()
+        assert 'b75eb90f68aa8ee1e29f2c542e851d3945066c54' == z.hexdigest()
 
         # Check basic 1-D array with some data
         z = self.create_array(shape=(1050,), chunks=100, dtype='<i4')
@@ -1899,7 +1899,7 @@ def test_hexdigest(self):
 
         # Check basic 2-D array
         z = self.create_array(shape=(20, 35,), chunks=10, dtype='<i4')
-        assert 'f57a9a73a4004490fe1b871688651b8a298a5db7' == z.hexdigest()
+        assert '37c7c46e5730bba37da5e518c9d75f0d774c5098' == z.hexdigest()
 
         # Check basic 1-D array with some data
         z = self.create_array(shape=(1050,), chunks=100, dtype='<i4')
@@ -1935,7 +1935,7 @@ def test_hexdigest(self):
 
         # Check basic 2-D array
         z = self.create_array(shape=(20, 35,), chunks=10, dtype='<i4')
-        assert 'deb675ff91dd26dba11b65aab5f19a1f21a5645b' == z.hexdigest()
+        assert '74ed339cfe84d544ac023d085ea0cd6a63f56c4b' == z.hexdigest()
 
         # Check basic 1-D array with some data
         z = self.create_array(shape=(1050,), chunks=100, dtype='<i4')
@@ -1978,7 +1978,7 @@ def test_hexdigest(self):
 
         # Check basic 2-D array
         z = self.create_array(shape=(20, 35,), chunks=10, dtype='<i4')
-        assert 'b93b163a21e8500519250a6defb821d03eb5d9e0' == z.hexdigest()
+        assert '9de97b5c49b38e68583ed701d7e8f4c94b6a8406' == z.hexdigest()
 
         # Check basic 1-D array with some data
         z = self.create_array(shape=(1050,), chunks=100, dtype='<i4')
@@ -2021,7 +2021,7 @@ def test_hexdigest(self):
 
         # Check basic 2-D array
         z = self.create_array(shape=(20, 35,), chunks=10, dtype='<i4')
-        assert '9abf3ad54413ab11855d88a5e0087cd416657e02' == z.hexdigest()
+        assert '7300f1eb130cff5891630038fd99c28ef23d3a01' == z.hexdigest()
 
         # Check basic 1-D array with some data
         z = self.create_array(shape=(1050,), chunks=100, dtype='<i4')

zarr/tests/test_sync.py

@@ -124,7 +124,7 @@ def test_hexdigest(self):
 
         # Check basic 2-D array
         z = self.create_array(shape=(20, 35,), chunks=10, dtype='<i4')
-        assert 'dde44c72cc530bd6aae39b629eb15a2da627e5f9' == z.hexdigest()
+        assert '6c530b6b9d73e108cc5ee7b6be3d552cc994bdbe' == z.hexdigest()
 
         # Check basic 1-D array with some data
         z = self.create_array(shape=(1050,), chunks=100, dtype='<i4')
@@ -168,7 +168,7 @@ def test_hexdigest(self):
 
         # Check basic 2-D array
         z = self.create_array(shape=(20, 35,), chunks=10, dtype='<i4')
-        assert 'dde44c72cc530bd6aae39b629eb15a2da627e5f9' == z.hexdigest()
+        assert '6c530b6b9d73e108cc5ee7b6be3d552cc994bdbe' == z.hexdigest()
 
         # Check basic 1-D array with some data
         z = self.create_array(shape=(1050,), chunks=100, dtype='<i4')

zarr/tests/test_util.py

@@ -29,7 +29,7 @@ def test_normalize_chunks():
     assert (10, 10) == normalize_chunks((10, 10), (100, 10), 1)
     assert (10, 10) == normalize_chunks(10, (100, 10), 1)
     assert (10, 10) == normalize_chunks((10, None), (100, 10), 1)
-    assert (30, 20, 10) == normalize_chunks(30, (100, 20, 10), 1)
+    assert (30, 30, 30) == normalize_chunks(30, (100, 20, 10), 1)
     assert (30, 20, 10) == normalize_chunks((30,), (100, 20, 10), 1)
     assert (30, 20, 10) == normalize_chunks((30, None), (100, 20, 10), 1)
     assert (30, 20, 10) == normalize_chunks((30, None, None), (100, 20, 10), 1)
@@ -41,8 +41,9 @@ def test_normalize_chunks():
         normalize_chunks((100, 10), (100,), 1)
 
     # test auto-chunking
-    chunks = normalize_chunks(None, (100,), 1)
-    assert (100,) == chunks
+    assert (100,) == normalize_chunks(None, (100,), 1)
+    assert (100,) == normalize_chunks(-1, (100,), 1)
+    assert (30, 20, 10) == normalize_chunks((30, -1, None), (100, 20, 10), 1)
 
 
 def test_is_total_slice():

zarr/util.py

@@ -128,7 +128,7 @@ def normalize_chunks(chunks, shape, typesize):
 
     # handle 1D convenience form
     if isinstance(chunks, numbers.Integral):
-        chunks = (int(chunks),)
+        chunks = tuple(int(chunks) for _ in shape)
 
     # handle bad dimensionality
     if len(chunks) > len(shape):
@@ -139,11 +139,12 @@ def normalize_chunks(chunks, shape, typesize):
         # assume chunks across remaining dimensions
         chunks += shape[len(chunks):]
 
-    # handle None in chunks
-    chunks = tuple(s if c is None else int(c)
-                   for s, c in zip(shape, chunks))
+    # handle None or -1 in chunks
+    if -1 in chunks or None in chunks:
+        chunks = tuple(s if c == -1 or c is None else int(c)
+                       for s, c in zip(shape, chunks))
 
-    return chunks
+    return tuple(chunks)
 
 
 def normalize_dtype(dtype, object_codec):