migrate_1to2

Author: alimanfoo

tox.ini

@@ -12,7 +12,7 @@ setenv =
 commands =
     python setup.py build_ext --inplace
     py27: nosetests -v zarr
-    py34,py35: nosetests -v --with-coverage --cover-erase --cover-min-percentage=100 --cover-package=zarr --with-doctest --doctest-options=+NORMALIZE_WHITESPACE zarr
+    py34,py35: nosetests -v --ignore-files=meta_v1.py --with-coverage --cover-erase --cover-min-percentage=100 --cover-package=zarr --with-doctest --doctest-options=+NORMALIZE_WHITESPACE zarr
     py34,py35: python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst
     py35: flake8 zarr
     python setup.py bdist_wheel

zarr/meta_v1.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, print_function, division
+import json
+
+
+import numpy as np
+
+
+from zarr.compat import PY2, text_type
+from zarr.errors import MetadataError
+
+
+def decode_metadata(b):
+    s = text_type(b, 'ascii')
+    meta = json.loads(s)
+    zarr_format = meta.get('zarr_format', None)
+    if zarr_format != 1:
+        raise MetadataError('unsupported zarr format: %s' % zarr_format)
+    try:
+        meta = dict(
+            zarr_format=meta['zarr_format'],
+            shape=tuple(meta['shape']),
+            chunks=tuple(meta['chunks']),
+            dtype=decode_dtype(meta['dtype']),
+            compression=meta['compression'],
+            compression_opts=meta['compression_opts'],
+            fill_value=meta['fill_value'],
+            order=meta['order'],
+        )
+    except Exception as e:
+        raise MetadataError('error decoding metadata: %s' % e)
+    else:
+        return meta
+
+
+def encode_metadata(meta):
+    meta = dict(
+        zarr_format=1,
+        shape=meta['shape'],
+        chunks=meta['chunks'],
+        dtype=encode_dtype(meta['dtype']),
+        compression=meta['compression'],
+        compression_opts=meta['compression_opts'],
+        fill_value=meta['fill_value'],
+        order=meta['order'],
+    )
+    s = json.dumps(meta, indent=4, sort_keys=True, ensure_ascii=True)
+    b = s.encode('ascii')
+    return b
+
+
+def encode_dtype(d):
+    if d.fields is None:
+        return d.str
+    else:
+        return d.descr
+
+
+def _decode_dtype_descr(d):
+    # need to convert list of lists to list of tuples
+    if isinstance(d, list):
+        # recurse to handle nested structures
+        if PY2:  # pragma: no cover
+            # under PY2 numpy rejects unicode field names
+            d = [(f.encode('ascii'), _decode_dtype_descr(v))
+                 for f, v in d]
+        else:
+            d = [(f, _decode_dtype_descr(v)) for f, v in d]
+    return d
+
+
+def decode_dtype(d):
+    d = _decode_dtype_descr(d)
+    return np.dtype(d)

zarr/storage.py

@@ -769,3 +769,32 @@ def getsize(self, path=None):
                     raise ValueError('path not found: %r' % path)
             else:
                 return 0
+
+
+def migrate_1to2(store):
+    """Migrate array metadata in `store` from Zarr format version 1 to
+    version 2.
+
+    Parameters
+    ----------
+    store : MutableMapping
+        Store to be migrated.
+
+    Notes
+    -----
+    Version 1 did not support hierarchies, so this migration function will
+    look for a single array in `store` and migrate the array metadata to
+    version 2.
+
+    """
+
+    # migrate metadata
+    from zarr import meta_v1
+    meta = meta_v1.decode_metadata(store['meta'])
+    del store['meta']
+    meta['filters'] = None
+    store[array_meta_key] = encode_array_metadata(meta)
+
+    # migrate user attributes
+    store[attrs_key] = store['attrs']
+    del store['attrs']

zarr/tests/test_storage.py

@@ -15,7 +15,7 @@
 
 
 from zarr.storage import init_array, array_meta_key, attrs_key, DictStore, \
-    DirectoryStore, ZipStore, init_group, group_meta_key, getsize
+    DirectoryStore, ZipStore, init_group, group_meta_key, getsize, migrate_1to2
 from zarr.meta import decode_array_metadata, encode_array_metadata, \
     ZARR_FORMAT, decode_group_metadata, encode_group_metadata
 from zarr.compat import text_type
@@ -642,3 +642,42 @@ def test_getsize():
     store['baz/quux'] = b'ccccc'
     eq(7, getsize(store))
     eq(5, getsize(store, 'baz'))
+
+
+def test_migrate_1to2():
+    from zarr import meta_v1
+
+    # N.B., version 1 did not support hierarchies, so we only have to be
+    # concerned about migrating a single array at the root of the store
+
+    # setup
+    store = dict()
+    meta = dict(
+        shape=(100,),
+        chunks=(10,),
+        dtype=np.dtype('f4'),
+        compression='zlib',
+        compression_opts=1,
+        fill_value=None,
+        order='C'
+    )
+    meta_json = meta_v1.encode_metadata(meta)
+    store['meta'] = meta_json
+    store['attrs'] = json.dumps(dict()).encode('ascii')
+
+    # run migration
+    migrate_1to2(store)
+
+    # check results
+    assert 'meta' not in store
+    assert '.zarray' in store
+    assert 'attrs' not in store
+    assert '.zattrs' in store
+    meta_migrated = decode_array_metadata(store['.zarray'])
+    eq(2, meta_migrated['zarr_format'])
+    # preserved fields
+    for f in 'shape', 'chunks', 'dtype', 'compression', 'compression_opts', \
+             'fill_value', 'order':
+        eq(meta[f], meta_migrated[f])
+    # TODO migrate should have added empty filters field
+    # assert_is_none(meta_migrated['filters'])