Merge pull request #459 from jrbourbeau/case-insensitive-directorystore

Add key normalization to DirectoryStore and subclass stores

Author: jrbourbeau

docs/release.rst

@@ -4,8 +4,12 @@ Release notes
 Upcoming Release
 ----------------
 
+* Add key normalization option for ``DirectoryStore``, ``NestedDirectoryStore``,
+  ``TempStore``, and ``N5Store``.
+  By :user:`James Bourbeau <jrbourbeau>`; :issue:`459`.
+
 * Add ``recurse`` keyword to ``Group.array_keys`` and ``Group.arrays`` methods.
-  By :user:`James Bourbeau <jrbourbeau>`; :issue:`458`
+  By :user:`James Bourbeau <jrbourbeau>`; :issue:`458`.
 
 * Use uniform chunking for all dimensions when specifying ``chunks`` as an integer.
   Also adds support for specifying ``-1`` to chunk across an entire dimension.

zarr/n5.py

@@ -35,6 +35,11 @@ class N5Store(NestedDirectoryStore):
     ----------
     path : string
         Location of directory to use as the root of the storage hierarchy.
+    normalize_keys : bool, optional
+        If True, all store keys will be normalized to use lower case characters
+        (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be
+        useful to avoid potential discrepancies between case-senstive and
+        case-insensitive file system. Default value is False.
 
     Examples
     --------

zarr/storage.py

@@ -682,6 +682,11 @@ class DirectoryStore(MutableMapping):
     ----------
     path : string
         Location of directory to use as the root of the storage hierarchy.
+    normalize_keys : bool, optional
+        If True, all store keys will be normalized to use lower case characters
+        (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be
+        useful to avoid potential discrepancies between case-senstive and
+        case-insensitive file system. Default value is False.
 
     Examples
     --------
@@ -728,16 +733,21 @@ class DirectoryStore(MutableMapping):
 
     """
 
-    def __init__(self, path):
+    def __init__(self, path, normalize_keys=False):
 
         # guard conditions
         path = os.path.abspath(path)
         if os.path.exists(path) and not os.path.isdir(path):
             err_fspath_exists_notdir(path)
 
         self.path = path
+        self.normalize_keys = normalize_keys
+
+    def _normalize_key(self, key):
+        return key.lower() if self.normalize_keys else key
 
     def __getitem__(self, key):
+        key = self._normalize_key(key)
         filepath = os.path.join(self.path, key)
         if os.path.isfile(filepath):
             with open(filepath, 'rb') as f:
@@ -746,6 +756,7 @@ def __getitem__(self, key):
             raise KeyError(key)
 
     def __setitem__(self, key, value):
+        key = self._normalize_key(key)
 
         # coerce to flat, contiguous array (ideally without copying)
         value = ensure_contiguous_ndarray(value)
@@ -785,6 +796,7 @@ def __setitem__(self, key, value):
                 os.remove(temp_path)
 
     def __delitem__(self, key):
+        key = self._normalize_key(key)
         path = os.path.join(self.path, key)
         if os.path.isfile(path):
             os.remove(path)
@@ -796,6 +808,7 @@ def __delitem__(self, key):
             raise KeyError(key)
 
     def __contains__(self, key):
+        key = self._normalize_key(key)
         file_path = os.path.join(self.path, key)
         return os.path.isfile(file_path)
 
@@ -910,14 +923,19 @@ class TempStore(DirectoryStore):
         Prefix for the temporary directory name.
     dir : string, optional
         Path to parent directory in which to create temporary directory.
+    normalize_keys : bool, optional
+        If True, all store keys will be normalized to use lower case characters
+        (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be
+        useful to avoid potential discrepancies between case-senstive and
+        case-insensitive file system. Default value is False.
 
     """
 
     # noinspection PyShadowingBuiltins
-    def __init__(self, suffix='', prefix='zarr', dir=None):
+    def __init__(self, suffix='', prefix='zarr', dir=None, normalize_keys=False):
         path = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir)
         atexit.register(atexit_rmtree, path)
-        super(TempStore, self).__init__(path)
+        super(TempStore, self).__init__(path, normalize_keys=normalize_keys)
 
 
 _prog_ckey = re.compile(r'^(\d+)(\.\d+)+$')
@@ -944,6 +962,11 @@ class NestedDirectoryStore(DirectoryStore):
     ----------
     path : string
         Location of directory to use as the root of the storage hierarchy.
+    normalize_keys : bool, optional
+        If True, all store keys will be normalized to use lower case characters
+        (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be
+        useful to avoid potential discrepancies between case-senstive and
+        case-insensitive file system. Default value is False.
 
     Examples
     --------
@@ -1000,8 +1023,8 @@ class NestedDirectoryStore(DirectoryStore):
 
     """
 
-    def __init__(self, path):
-        super(NestedDirectoryStore, self).__init__(path)
+    def __init__(self, path, normalize_keys=False):
+        super(NestedDirectoryStore, self).__init__(path, normalize_keys=normalize_keys)
 
     def __getitem__(self, key):
         key = _nested_map_ckey(key)

zarr/tests/test_storage.py

@@ -735,10 +735,10 @@ def test_deprecated(self):
 
 class TestDirectoryStore(StoreTests, unittest.TestCase):
 
-    def create_store(self):
+    def create_store(self, normalize_keys=False):
         path = tempfile.mkdtemp()
         atexit.register(atexit_rmtree, path)
-        store = DirectoryStore(path)
+        store = DirectoryStore(path, normalize_keys=normalize_keys)
         return store
 
     def test_filesystem_path(self):
@@ -782,13 +782,19 @@ def test_setdel(self):
         store = self.create_store()
         setdel_hierarchy_checks(store)
 
+    def test_normalize_keys(self):
+        store = self.create_store(normalize_keys=True)
+        store['FOO'] = b'bar'
+        assert 'FOO' in store
+        assert 'foo' in store
+
 
 class TestNestedDirectoryStore(TestDirectoryStore, unittest.TestCase):
 
-    def create_store(self):
+    def create_store(self, normalize_keys=False):
         path = tempfile.mkdtemp()
         atexit.register(atexit_rmtree, path)
-        store = NestedDirectoryStore(path)
+        store = NestedDirectoryStore(path, normalize_keys=normalize_keys)
         return store
 
     def test_chunk_nesting(self):
@@ -806,10 +812,10 @@ def test_chunk_nesting(self):
 
 class TestN5Store(TestNestedDirectoryStore, unittest.TestCase):
 
-    def create_store(self):
+    def create_store(self, normalize_keys=False):
         path = tempfile.mkdtemp(suffix='.n5')
         atexit.register(atexit_rmtree, path)
-        store = N5Store(path)
+        store = N5Store(path, normalize_keys=normalize_keys)
         return store
 
     def test_equal(self):