Merge pull request #4 from marqh/netcdf

Netcdf

Author: marqh

.gitignore

@@ -15,7 +15,6 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
 lib64/
 parts/
 sdist/

lib/bald/__init__.py

@@ -2,9 +2,10 @@
 import re
 
 import h5py
+import netCDF4
 import requests
 
-from bald.validation import ContainerValidation, DatasetValidation
+import bald.validation as bv
 
 
 class HttpCache(object):
@@ -19,18 +20,18 @@ def __getitem__(self, item):
         if not item.startswith('http://') or item.startswith('https://'):
             raise ValueError('{} is not a HTTP URI.'.format(item))
         if item not in self.cache:
-            headers={'Accept':'text/turtle'}
+            headers = {'Accept': 'text/turtle'}
             self.cache[item] = requests.get(item, headers=headers)
 
         return self.cache[item]
-        
 
     def check_uri(self, uri):
         result = False
         if self[uri].status_code == 200:
             result = True
         return result
 
+
 class Subject(object):
     def __init__(self, attrs=None):
         """
@@ -46,32 +47,47 @@ def __init__(self, attrs=None):
         self._http_uri = re.compile('{}'.format(_http_p))
         self._http_uri_prefix = re.compile('{}/|#'.format(_http_p))
 
-
     def prefixes(self):
         prefixes = {}
         for key, value in self.attrs.iteritems():
             if key.endswith('__') and self._http_uri_prefix.match(value):
                 pref = key.rstrip('__')
-                if prefixes.has_key(pref):
-                    raise ValueError('This container has conflicting prefix definitions')
+                if pref in prefixes:
+                    raise ValueError('This container has conflicting prefix'
+                                     ' definitions.')
                 prefixes[pref] = value
         return prefixes
 
-
     def unpack_uri(self, astring):
         result = astring
         if self._prefix_suffix.match(astring):
             prefix, suffix = self._prefix_suffix.match(astring).groups()
-
-            if self.prefixes().has_key(prefix):
+            if prefix in self.prefixes():
                 if self._http_uri.match(self.prefixes()[prefix]):
-                    result = astring.replace('{}__'.format(prefix), self.prefixes()[prefix])
+                    result = astring.replace('{}__'.format(prefix),
+                                             self.prefixes()[prefix])
         return result
 
+
+@contextlib.contextmanager
+def load(afilepath):
+    if afilepath.endswith('.hdf'):
+        loader = h5py.File
+    else:
+        raise ValueError('filepath suffix not supported')
+    try:
+        f = loader(afilepath, "r")
+        yield f
+    finally:
+        f.close()
+
+
 @contextlib.contextmanager
 def load(afilepath):
     if afilepath.endswith('.hdf'):
         loader = h5py.File
+    elif afilepath.endswith('.nc'):
+        loader = netCDF4.Dataset
     else:
         raise ValueError('filepath suffix not supported')
     try:
@@ -81,32 +97,56 @@ def load(afilepath):
         f.close()
 
 
+def validate_netcdf(afilepath):
+    """
+    Validate a file with respect to binarry-array-linked-data.
+    Returns a :class:`bald.validation.Validation`
+    """
+
+    with load(afilepath) as fhandle:
+        sval = bv.StoredValidation()
+        attrs = {}
+        for k in fhandle.ncattrs():
+            attrs[k] = getattr(fhandle, k)
+        root_container = Subject(attrs)
+        root_val = bv.ContainerValidation(subject=root_container,
+                                          fhandle=fhandle)
+        sval.stored_exceptions += root_val.exceptions()
+        for name in fhandle.variables:
+            sattrs = fhandle.__dict__.copy()
+            sattrs.update(fhandle.variables[name].__dict__.copy())
+            var = Subject(sattrs)
+            var_val = bv.ArrayValidation(name, fhandle.variables[name], fhandle=fhandle,
+                                         subject=var)
+            sval.stored_exceptions += var_val.exceptions()
+            
+        return sval
+
+
 def validate_hdf5(afilepath):
     """
     Validate a file with respect ot binarry-array-linked-data.
-    Returns a :class:`bald.validation.Validation` 
+    Returns a :class:`bald.validation.Validation`
     """
-    
+
     with load(afilepath) as fhandle:
-        valid = True
+        sval = bv.StoredValidation()
         cache = {}
         root_container = Subject(fhandle.attrs)
-        root_val = ContainerValidation(subject=root_container, fhandle=fhandle)
-        if not root_val.is_valid():
-            valid = False
+        root_val = bv.ContainerValidation(subject=root_container,
+                                          fhandle=fhandle)
+        sval.stored_exceptions += root_val.exceptions()
         # iterate through the datasets
         for name, dataset in fhandle.items():
-            # a dataset's attribute collection inherits from and specialises it's
-            # container's attrbiute collection
+            # a dataset's attribute collection inherits from and
+            # specialises it's container's attrbiute collection
+            # this only helps with prefixes, afaik, hence:
+            # #
             sattrs = dict(fhandle.attrs).copy()
             sattrs.update(dataset.attrs)
             dset = Subject(sattrs)
-            dset_val = DatasetValidation(name, dataset, fhandle=fhandle, subject=dset)
-            if not dset_val.is_valid():
-                valid = False
-        
-        return valid
-        
-        
-
-    
+            dset_val = bv.ArrayValidation(name, dataset, fhandle=fhandle,
+                                            subject=dset)
+            sval.stored_exceptions += dset_val.exceptions()
+
+        return sval

lib/bald/tests/__init__.pyc

@@ -0,0 +1,78 @@
+import unittest
+
+import h5py
+import netCDF4
+import numpy as np
+
+import bald
+from bald.tests import BaldTestCase
+
+def _fattrs(f):
+    f.bald__ = 'http://binary-array-ld.net/latest/'
+    f.rdf__ = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'
+    f.rdf__type =  'bald__Container'
+    return f
+
+def _create_parent_child(f, pshape, cshape):
+    for i, pdimsize in enumerate(pshape):
+        f.createDimension("pdim{}".format(str(i)), pdimsize)
+    for i, cdimsize in enumerate(cshape):
+        f.createDimension("cdim{}".format(str(i)), cdimsize)
+    varp = f.createVariable("parent_variable", 'i4', tuple(["pdim{}".format(str(i)) for i, _ in enumerate(pshape)]))
+    varc = f.createVariable("child_variable", 'i4', tuple(["cdim{}".format(str(i)) for i, _ in enumerate(cshape)]))
+    varp.rdf__type = 'bald__Array'
+    varp.bald__references = "child_variable"
+    varc.rdf__type = 'bald__Array'
+    varc.rdf__type = 'bald__Reference'
+    varc.bald__array = "child_variable"
+    return f
+
+
+class Test(BaldTestCase):
+        
+    def test_valid_uri(self):
+        with self.temp_filename('.nc') as tfile:
+            f = netCDF4.Dataset(tfile, "w", format="NETCDF4")
+
+            f = _fattrs(f)
+            f.close()
+            validation = bald.validate_netcdf(tfile)
+            self.assertTrue(validation.is_valid())
+
+    def test_invalid_uri(self):
+        with self.temp_filename('.nc') as tfile:
+            f = netCDF4.Dataset(tfile, "w", format="NETCDF4")
+
+            f = _fattrs(f)
+            setattr(f, 'bald__turtle', 'bald__walnut')
+            f.close()
+            validation = bald.validate_netcdf(tfile)
+            self.assertFalse(validation.is_valid())
+
+
+class TestArrayReference(BaldTestCase):
+    def test_match(self):
+        with self.temp_filename('.nc') as tfile:
+            f = netCDF4.Dataset(tfile, "w", format="NETCDF4")
+            f = _fattrs(f)
+            f = _create_parent_child(f, (11, 17), (11, 17))
+            f.close()
+            validation = bald.validate_netcdf(tfile)
+            self.assertTrue(validation.is_valid())
+
+    def test_mismatch_zeroth(self):
+        with self.temp_filename('.nc') as tfile:
+            f = netCDF4.Dataset(tfile, "w", format="NETCDF4")
+            f = _fattrs(f)
+            f = _create_parent_child(f, (11, 17), (11, 13))
+            f.close()
+            validation = bald.validate_netcdf(tfile)
+            self.assertFalse(validation.is_valid())
+
+
+if __name__ == '__main__':
+    unittest.main()
+
+
+
+

lib/bald/tests/integration/test_netcdf.py

@@ -7,19 +7,19 @@
 from bald.tests import BaldTestCase
 
 def _fattrs(f):
-    f.attrs['bald__'] = 'http://binary-array-ld.net/experimental/'
+    f.attrs['bald__'] = 'http://binary-array-ld.net/latest/'
     f.attrs['rdf__'] = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'
     f.attrs['rdf__type'] = 'bald__Container'
     return f
 
 def _create_parent_child(f, pshape, cshape):
     dsetp = f.create_dataset("parent_dataset", pshape, dtype='i')
     dsetc = f.create_dataset("child_dataset", cshape, dtype='i')
-    dsetp.attrs['rdf__type'] = 'bald__Dataset'
+    dsetp.attrs['rdf__type'] = 'bald__Array'
     dsetp.attrs['bald__references'] = dsetc.ref
-    dsetc.attrs['rdf__type'] = 'bald__Dataset'
+    dsetc.attrs['rdf__type'] = 'bald__Array'
     dsetc.attrs['rdf__type'] = 'bald__Reference'
-    dsetc.attrs['bald__dataset'] = dsetc.ref
+    dsetc.attrs['bald__array'] = dsetc.ref
     return f
 
 
@@ -31,7 +31,8 @@ def test_valid_uri(self):
             f = _fattrs(f)
             f = _create_parent_child(f, (11, 17), (11, 17))
             f.close()
-            self.assertTrue(bald.validate_hdf5(tfile))
+            validation = bald.validate_hdf5(tfile)
+            self.assertTrue(validation.is_valid())
 
     def test_invalid_uri(self):
         with self.temp_filename('.hdf') as tfile:
@@ -40,7 +41,8 @@ def test_invalid_uri(self):
             f = _create_parent_child(f, (11, 17), (11, 17))
             f.attrs['bald__turtle'] = 'bald__walnut'
             f.close()
-            self.assertFalse(bald.validate_hdf5(tfile))
+            validation = bald.validate_hdf5(tfile)
+            self.assertFalse(validation.is_valid())
 
 class TestArrayReference(BaldTestCase):
     def test_match(self):
@@ -49,23 +51,26 @@ def test_match(self):
             f = _fattrs(f)
             f = _create_parent_child(f, (11, 17), (11, 17))
             f.close()
-            self.assertTrue(bald.validate_hdf5(tfile))
+            validation = bald.validate_hdf5(tfile)
+            self.assertTrue(validation.is_valid())
 
     def test_mismatch_zeroth(self):
         with self.temp_filename('.hdf') as tfile:
             f = h5py.File(tfile, "w")
             f = _fattrs(f)
             f = _create_parent_child(f, (11, 17), (11, 13))
             f.close()
-            self.assertFalse(bald.validate_hdf5(tfile))
+            validation = bald.validate_hdf5(tfile)
+            self.assertFalse(validation.is_valid())
 
     def test_mismatch_oneth(self):
         with self.temp_filename('.hdf') as tfile:
             f = h5py.File(tfile, "w")
             f = _fattrs(f)
             f = _create_parent_child(f, (11, 17), (13, 17))
             f.close()
-            self.assertFalse(bald.validate_hdf5(tfile))
+            validation = bald.validate_hdf5(tfile)
+            self.assertFalse(validation.is_valid())
 
     def test_match_plead_dim(self):
         with self.temp_filename('.hdf') as tfile:
@@ -74,7 +79,8 @@ def test_match_plead_dim(self):
             # parent has leading dimension wrt child
             f = _create_parent_child(f, (4, 13, 17), (13, 17))
             f.close()
-            self.assertTrue(bald.validate_hdf5(tfile))
+            validation = bald.validate_hdf5(tfile)
+            self.assertTrue(validation.is_valid())
 
     def test_match_clead_dim(self):
         with self.temp_filename('.hdf') as tfile:
@@ -83,7 +89,8 @@ def test_match_clead_dim(self):
             # child has leading dimension wrt parent
             f = _create_parent_child(f, (13, 17), (7, 13, 17))
             f.close()
-            self.assertTrue(bald.validate_hdf5(tfile))
+            validation = bald.validate_hdf5(tfile)
+            self.assertTrue(validation.is_valid())
 
     def test_mismatch_pdisjc_lead_dim(self):
         with self.temp_filename('.hdf') as tfile:
@@ -93,7 +100,8 @@ def test_mismatch_pdisjc_lead_dim(self):
             f = _create_parent_child(f, (4, 13, 17), (7, 13, 17))
 
             f.close()
-            self.assertFalse(bald.validate_hdf5(tfile))
+            validation = bald.validate_hdf5(tfile)
+            self.assertFalse(validation.is_valid())
 
     def test_mismatch_pdisjc_trail_dim(self):
         with self.temp_filename('.hdf') as tfile:
@@ -102,8 +110,8 @@ def test_mismatch_pdisjc_trail_dim(self):
             # child and parent have disjoint trailing dimensions
             f = _create_parent_child(f, (13, 17, 2), (13, 17, 9))
             f.close()
-            self.assertFalse(bald.validate_hdf5(tfile))
-
+            validation = bald.validate_hdf5(tfile)
+            self.assertFalse(validation.is_valid())
 
 
     # def test_match_(self):
@@ -113,7 +121,7 @@ def test_mismatch_pdisjc_trail_dim(self):
     #         # 
     #         f = _create_parent_child(f, (), ())
     #         f.close()
-    #         self.assert(bald.validate_hdf5(tfile))
+    #         self.assert(bald.validate_hdf5(tfile).is_valid())
 
 if __name__ == '__main__':
     unittest.main()