caching

Author: marqh

lib/bald/__init__.py

@@ -2,9 +2,41 @@
 import re
 
 import h5py
+import requests
 
 from bald.validation import ContainerValidation, DatasetValidation
 
+
+class HttpStatusCache(object):
+    def __init__(self):
+        self.cache = {}
+
+    def __getitem__(self, item):
+        uri = item
+        if item not in self.cache:
+            if not uri.startswith('http://') or uri.startswith('https://'):
+                raise ValueError('{} is not a HTTP URI.'.format(item))
+        if item in self.cache:
+            result = self.cache[item]
+        else:
+            r = requests.get(uri)
+            if r.status_code == 200:
+                headers={'Accept':'text/turtle'}
+                rraw = requests.get(uri, headers=headers)
+                self.cache[item] = rraw.status_code
+            else:
+                self.cache[item] = r.status_code
+
+            result = self.cache[item]
+        return result
+        
+
+    def check_uri(self, uri):
+        result = False
+        if self[uri] == 200:
+            result = True
+        return result
+
 class Subject(object):
     def __init__(self, attrs=None):
         """
@@ -61,14 +93,19 @@ def validate_hdf5(afilepath):
 
     with load(afilepath) as fhandle:
         valid = True
+        cache = {}
         root_container = Subject(fhandle.attrs)
-        root_val = ContainerValidation(root_container)
+        root_val = ContainerValidation(subject=root_container)
         if not root_val.is_valid():
             valid = False
         # iterate through the datasets
         for name, dataset in fhandle.items():
-            dset = Subject(dataset.attrs)
-            dset_val = DatasetValidation(name, dataset, dset, fhandle)
+            # a dataset's attribute collection inherits from and specialises it's
+            # container's attrbiute collection
+            sattrs = dict(fhandle.attrs).copy()
+            sattrs.update(dataset.attrs)
+            dset = Subject(sattrs)
+            dset_val = DatasetValidation(name, dataset, fhandle, subject=dset)
             if not dset_val.is_valid():
                 valid = False
 

lib/bald/tests/integration/test_validation.py

@@ -40,30 +40,76 @@ def test_invalid_uri(self):
             self.assertFalse(bald.validate_hdf5(tfile))
 
 class TestArrayReference(BaldTestCase):
-    def test_match_array_reference(self):
+    def test_match(self):
         with self.temp_filename('.hdf') as tfile:
             f = h5py.File(tfile, "w")
             f = _fattrs(f)
             f = _create_parent_child(f, (11, 17), (11, 17))
             f.close()
             self.assertTrue(bald.validate_hdf5(tfile))
 
-    def test_misatch_zeroth_array_reference(self):
+    def test_mismatch_zeroth(self):
         with self.temp_filename('.hdf') as tfile:
             f = h5py.File(tfile, "w")
             f = _fattrs(f)
             f = _create_parent_child(f, (11, 17), (11, 13))
             f.close()
             self.assertFalse(bald.validate_hdf5(tfile))
 
-    def test_misatch_oneth_array_reference(self):
+    def test_mismatch_oneth(self):
         with self.temp_filename('.hdf') as tfile:
             f = h5py.File(tfile, "w")
             f = _fattrs(f)
             f = _create_parent_child(f, (11, 17), (13, 17))
             f.close()
             self.assertFalse(bald.validate_hdf5(tfile))
 
+    def test_match_plead_dim(self):
+        with self.temp_filename('.hdf') as tfile:
+            f = h5py.File(tfile, "w")
+            f = _fattrs(f)
+            # parent has leading dimension wrt child
+            f = _create_parent_child(f, (4, 13, 17), (13, 17))
+            f.close()
+            self.assertTrue(bald.validate_hdf5(tfile))
+
+    def test_match_clead_dim(self):
+        with self.temp_filename('.hdf') as tfile:
+            f = h5py.File(tfile, "w")
+            f = _fattrs(f)
+            # child has leading dimension wrt parent
+            f = _create_parent_child(f, (13, 17), (7, 13, 17))
+            f.close()
+            self.assertTrue(bald.validate_hdf5(tfile))
+
+    def test_mismatch_pdisjc_lead_dim(self):
+        with self.temp_filename('.hdf') as tfile:
+            f = h5py.File(tfile, "w")
+            f = _fattrs(f)
+            # child and parent have disjoint leading dimensions
+            f = _create_parent_child(f, (4, 13, 17), (7, 13, 17))
+            f.close()
+            self.assertFalse(bald.validate_hdf5(tfile))
+
+    def test_mismatch_pdisjc_trail_dim(self):
+        with self.temp_filename('.hdf') as tfile:
+            f = h5py.File(tfile, "w")
+            f = _fattrs(f)
+            # child and parent have disjoint trailing dimensions
+            f = _create_parent_child(f, (13, 17, 2), (13, 17, 9))
+            f.close()
+            self.assertFalse(bald.validate_hdf5(tfile))
+
+
+
+    # def test_match_(self):
+    #      with self.temp_filename('.hdf') as tfile:
+    #         f = h5py.File(tfile, "w")
+    #         f = _fattrs(f)
+    #         # 
+    #         f = _create_parent_child(f, (), ())
+    #         f.close()
+    #         self.assert(bald.validate_hdf5(tfile))
 
 if __name__ == '__main__':
     unittest.main()

lib/bald/validation.py

@@ -1,19 +1,8 @@
 
 import numpy as np
-import requests
 
 import bald
 
-def check_uri(uri):
-    result = False
-    if uri.startswith('http://') or uri.startswith('https://'):
-        r = requests.get(uri)
-        if r.status_code == 200:
-            headers={'Accept':'text/turtle'}
-            rraw = requests.get(uri, headers=headers)
-            if rraw.status_code == 200:
-                result = True
-    return result
 
 def valid_array_reference(parray, carray):
     """
@@ -51,8 +40,13 @@ def exceptions(self):
 
 class SubjectValidation(Validation):
 
-    def __init__(self, subject):
+    def __init__(self, subject, httpcache=None): 
         self.subject = subject
+        if isinstance(httpcache, bald.HttpStatusCache):
+            self.cache = httpcache
+        else:
+            self.cache = bald.HttpStatusCache()
+            
 
     def is_valid(self):
         return not self.exceptions()
@@ -64,8 +58,7 @@ def exceptions(self):
 
     def check_attr_uris(self, exceptions):
         def _check_uri(uri, exceptions):
-            if not check_uri(uri):
-                check_uri(uri)
+            if not self.cache.check_uri(uri):
                 msg = '{} is not resolving as a resource (404).'
                 msg = msg.format(uri)
                 exceptions.append(ValueError(msg))
@@ -85,9 +78,8 @@ def _check_uri(uri, exceptions):
 
 class ContainerValidation(SubjectValidation):
 
-    def __init__(self, container):
-        self.container = container
-        self.subject = container
+    def __init__(self, **kwargs):
+        super(ContainerValidation, self).__init__(**kwargs)
 
     def exceptions(self):
         exceptions = []
@@ -97,11 +89,12 @@ def exceptions(self):
 class DatasetValidation(SubjectValidation):
 
 
-    def __init__(self, name, dataset, subject, fhandle):
+    def __init__(self, name, dataset, fhandle, **kwargs):
+        
         self.dataset = dataset
-        self.subject = subject
         self.name = name
         self.fhandle = fhandle
+        super(DatasetValidation, self).__init__(**kwargs)
 
     def exceptions(self):
         exceptions = []