Merge pull request #1 from marqh/referCache

Refer cache

Author: marqh

lib/bald/__init__.py

@@ -235,11 +235,15 @@ def __getitem__(self, item):
         if not self.is_http_uri(item):
             raise ValueError('{} is not a HTTP URI.'.format(item))
         if item not in self.cache:
-            headers = {'Accept': 'text/turtle'}
             # import datetime
             # now = datetime.datetime.utcnow()
             # print('\ndownloading: {}'.format(item))
-            self.cache[item] = requests.get(item, headers=headers)
+            try:
+                headers = {'Accept': 'application/rdf+xml'}
+                self.cache[item] = requests.get(item, headers=headers)
+            except Exception:
+                headers = {'Accept': 'text/html'}
+                self.cache[item] = requests.get(item, headers=headers)
             # then = datetime.datetime.utcnow()
             # print('{}s'.format((then-now).total_seconds()))
 
@@ -387,9 +391,6 @@ def unpack_rdfobject(self, astring, predicate):
             # qres = self.alias_graph.query(rdfobj_alias_query)
             try:
                 qres = self.alias_graph.query(rdfobj_alias_query)
-            # except Exception:
-            #     import pdb; pdb.set_trace()
-            #     qres = self.alias_graph.query(rdfobj_alias_query)
                 results = list(qres)
                 if len(results) > 1:
                     raise ValueError('multiple alias options')
@@ -637,13 +638,16 @@ def load(afilepath):
     finally:
         f.close()
 
-def load_netcdf(afilepath, baseuri=None, alias_dict=None):
+def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None):
     """
     Load a file with respect to binary-array-linked-data.
     Returns a :class:`bald.Collection`
     """
     if alias_dict == None:
         alias_dict = {}
+    if cache is None:
+        cache = HttpCache()
+
     with load(afilepath) as fhandle:
         if baseuri is None:
             baseuri = 'file://{}'.format(afilepath)
@@ -693,11 +697,21 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None):
             attrs[k] = getattr(fhandle, k)
 
         aliasgraph = rdflib.Graph()
+
         for alias in aliases:
+            response = cache[aliases[alias]]
             try:
-                aliasgraph.parse(aliases[alias], format='xml')
-            except TypeError:
-                pass
+                aliasgraph.parse(data=response.text, format='xml')
+            except Exception:
+                print('Failed to parse: {}'.format(aliases[alias]))
+            # try:
+            #     import xml.sax._exceptions
+            #     aliasgraph.parse(data=response.text, format='xml')
+            # except TypeError:
+            #     pass
+            # except xml.sax._exceptions.SAXParseException:
+            #     import pdb; pdb.set_trace()
+            #     pass
         # if hasattr(fhandle, 'Conventions'):
         #     conventions = [c.strip() for c in fhandle.Conventions.split(',')]
         #     for conv in conventions:
@@ -715,7 +729,6 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None):
             # if len(set(na_keys)) != len(na_keys):
             #     raise ValueError('duplicate aliases')
             # aliases = careful_update(aliases, dict(new_aliases))
-                    
         root_container = Container(baseuri, '', attrs, prefixes=prefixes,
                                    aliases=aliases, alias_graph=aliasgraph)
 
@@ -750,7 +763,11 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None):
 
         reference_prefixes = dict()
         reference_graph = copy.copy(aliasgraph)
-        reference_graph.parse('http://binary-array-ld.net/latest?_format=ttl')
+
+        response = cache['http://binary-array-ld.net/latest']
+        reference_graph.parse(data=response.text, format='xml')
+
+        # reference_graph.parse('http://binary-array-ld.net/latest?_format=ttl')
         qstr = ('prefix bald: <http://binary-array-ld.net/latest/> '
                 'prefix skos: <http://www.w3.org/2004/02/skos/core#> '
                 'select ?s '
@@ -842,24 +859,24 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None):
     return root_container
 
 
-def validate_netcdf(afilepath, cache=None, baseuri=None):
+def validate_netcdf(afilepath, baseuri=None, cache=None):
     """
     Validate a file with respect to binary-array-linked-data.
     Returns a :class:`bald.validation.Validation`
 
     """
-    root_container = load_netcdf(afilepath, baseuri=baseuri)
+    root_container = load_netcdf(afilepath, baseuri=baseuri, cache=cache)
     return validate(root_container, cache=cache)
 
 
-def validate_hdf5(afilepath, cache=None, baseuri=None):
+def validate_hdf5(afilepath, baseuri=None, cache=None):
     """
     Validate a file with respect to binary-array-linked-data.
     Returns a :class:`bald.validation.Validation`
 
     """
-    root_container = load_hdf5(afilepath, baseuri=baseuri)
-    return validate(root_container)
+    root_container = load_hdf5(afilepath, baseuri=baseuri, cache=cache)
+    return validate(root_container, cache=cache)
 
 def validate(root_container, sval=None, cache=None):
     """
@@ -896,20 +913,24 @@ def careful_update(adict, bdict):
         adict.update(bdict)
         return adict
 
-def load_hdf5(afilepath, baseuri=None, alias_dict=None):
+def load_hdf5(afilepath, baseuri=None, alias_dict=None, cache=None):
+    if cache is None:
+        cache = HttpCache()
     with load(afilepath) as fhandle:
         # unused?
         cache = {}
         if baseuri is None:
             baseuri = 'file://{}'.format(afilepath)
 
         root_container, file_variables = _hdf_group(fhandle, baseuri=baseuri,
-                                                    alias_dict=alias_dict)
+                                                    alias_dict=alias_dict, cache=cache)
         _hdf_references(fhandle, root_container, file_variables)
     return root_container
 
 def _hdf_group(fhandle, identity='root', baseuri=None, prefixes=None,
-               aliases=None, alias_dict=None):
+               aliases=None, alias_dict=None, cache=None):
+    if cache is None:
+        cache = HttpCache()
 
     prefix_group = fhandle.attrs.get('bald__isPrefixedBy')
     if prefixes is None:

lib/bald/tests/integration/HTML/hdf_container_nest.html

@@ -174,33 +174,33 @@
      return cell;
 };
 
-var file:///tmp/tmpk7nc3lox.hdf/alocation = instance('file:///tmp/tmpk7nc3lox.hdf/alocation:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>, <a xlink:href="http://binary-array-ld.net/latest/Reference" xlink:show=new text-decoration="underline">bald__Reference</a>', ['<a xlink:href="http://binary-array-ld.net/latest/array" xlink:show=new text-decoration="underline">bald__array</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 11, 17'], '#878800');
-var file:///tmp/tmpk7nc3lox.hdf/anotherpair = instance('file:///tmp/tmpk7nc3lox.hdf/anotherpair:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>, <a xlink:href="http://binary-array-ld.net/latest/Reference" xlink:show=new text-decoration="underline">bald__Reference</a>', ['<a xlink:href="http://binary-array-ld.net/latest/array" xlink:show=new text-decoration="underline">bald__array</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 2'], '#878800');
-var file:///tmp/tmpk7nc3lox.hdf/apair = instance('file:///tmp/tmpk7nc3lox.hdf/apair:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 2'], '#878800');
-var file:///tmp/tmpk7nc3lox.hdf/data = instance('file:///tmp/tmpk7nc3lox.hdf/data:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 11, 17'], '#878800');
-var file:///tmp/tmpk7nc3lox.hdf/discovery = instance('file:///tmp/tmpk7nc3lox.hdf/discovery:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
-var file:///tmp/tmpk7nc3lox.hdf/institution = instance('file:///tmp/tmpk7nc3lox.hdf/institution:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: a quality establishment'], '#878800');
-var file:///tmp/tmpk7nc3lox.hdf/locref = instance('file:///tmp/tmpk7nc3lox.hdf/locref:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: for locational purposes'], '#878800');
-var file:///tmp/tmpk7nc3lox.hdf/locref2 = instance('file:///tmp/tmpk7nc3lox.hdf/locref2:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: for more locational purposes'], '#878800');
-var file:///tmp/tmpk7nc3lox.hdf/referencing = instance('file:///tmp/tmpk7nc3lox.hdf/referencing:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
-var file:///tmp/tmpk7nc3lox.hdf/root = instance('file:///tmp/tmpk7nc3lox.hdf/root:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
-var file:///tmp/tmpk7nc3lox.hdf/source = instance('file:///tmp/tmpk7nc3lox.hdf/source:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
-link(file:///tmp/tmpk7nc3lox.hdf/alocation, file:///tmp/tmpk7nc3lox.hdf/alocation, 'bald__array', 'bottom');
-link(file:///tmp/tmpk7nc3lox.hdf/alocation, file:///tmp/tmpk7nc3lox.hdf/locref, 'bald__references');
-link(file:///tmp/tmpk7nc3lox.hdf/alocation, file:///tmp/tmpk7nc3lox.hdf/locref2, 'bald__references');
-link(file:///tmp/tmpk7nc3lox.hdf/anotherpair, file:///tmp/tmpk7nc3lox.hdf/anotherpair, 'bald__array', 'bottom');
-link(file:///tmp/tmpk7nc3lox.hdf/apair, file:///tmp/tmpk7nc3lox.hdf/anotherpair, 'bald__references');
-link(file:///tmp/tmpk7nc3lox.hdf/data, file:///tmp/tmpk7nc3lox.hdf/alocation, 'bald__references');
-link(file:///tmp/tmpk7nc3lox.hdf/discovery, file:///tmp/tmpk7nc3lox.hdf/anotherpair, 'bald__contains', 'top', true);
-link(file:///tmp/tmpk7nc3lox.hdf/discovery, file:///tmp/tmpk7nc3lox.hdf/apair, 'bald__contains', 'top', true);
-link(file:///tmp/tmpk7nc3lox.hdf/discovery, file:///tmp/tmpk7nc3lox.hdf/source, 'bald__contains', 'top', true);
-link(file:///tmp/tmpk7nc3lox.hdf/referencing, file:///tmp/tmpk7nc3lox.hdf/locref, 'bald__contains', 'top', true);
-link(file:///tmp/tmpk7nc3lox.hdf/referencing, file:///tmp/tmpk7nc3lox.hdf/locref2, 'bald__contains', 'top', true);
-link(file:///tmp/tmpk7nc3lox.hdf/root, file:///tmp/tmpk7nc3lox.hdf/alocation, 'bald__contains', 'top', true);
-link(file:///tmp/tmpk7nc3lox.hdf/root, file:///tmp/tmpk7nc3lox.hdf/data, 'bald__contains', 'top', true);
-link(file:///tmp/tmpk7nc3lox.hdf/root, file:///tmp/tmpk7nc3lox.hdf/discovery, 'bald__contains', 'top', true);
-link(file:///tmp/tmpk7nc3lox.hdf/root, file:///tmp/tmpk7nc3lox.hdf/referencing, 'bald__contains', 'top', true);
-link(file:///tmp/tmpk7nc3lox.hdf/source, file:///tmp/tmpk7nc3lox.hdf/institution, 'bald__contains', 'top', true);
+var file:///tmp/tmp0ed878yz.hdf/alocation = instance('file:///tmp/tmp0ed878yz.hdf/alocation:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>, <a xlink:href="http://binary-array-ld.net/latest/Reference" xlink:show=new text-decoration="underline">bald__Reference</a>', ['<a xlink:href="http://binary-array-ld.net/latest/array" xlink:show=new text-decoration="underline">bald__array</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 11, 17'], '#878800');
+var file:///tmp/tmp0ed878yz.hdf/anotherpair = instance('file:///tmp/tmp0ed878yz.hdf/anotherpair:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>, <a xlink:href="http://binary-array-ld.net/latest/Reference" xlink:show=new text-decoration="underline">bald__Reference</a>', ['<a xlink:href="http://binary-array-ld.net/latest/array" xlink:show=new text-decoration="underline">bald__array</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 2'], '#878800');
+var file:///tmp/tmp0ed878yz.hdf/apair = instance('file:///tmp/tmp0ed878yz.hdf/apair:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 2'], '#878800');
+var file:///tmp/tmp0ed878yz.hdf/data = instance('file:///tmp/tmp0ed878yz.hdf/data:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 11, 17'], '#878800');
+var file:///tmp/tmp0ed878yz.hdf/discovery = instance('file:///tmp/tmp0ed878yz.hdf/discovery:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
+var file:///tmp/tmp0ed878yz.hdf/institution = instance('file:///tmp/tmp0ed878yz.hdf/institution:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: a quality establishment'], '#878800');
+var file:///tmp/tmp0ed878yz.hdf/locref = instance('file:///tmp/tmp0ed878yz.hdf/locref:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: for locational purposes'], '#878800');
+var file:///tmp/tmp0ed878yz.hdf/locref2 = instance('file:///tmp/tmp0ed878yz.hdf/locref2:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: for more locational purposes'], '#878800');
+var file:///tmp/tmp0ed878yz.hdf/referencing = instance('file:///tmp/tmp0ed878yz.hdf/referencing:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
+var file:///tmp/tmp0ed878yz.hdf/root = instance('file:///tmp/tmp0ed878yz.hdf/root:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
+var file:///tmp/tmp0ed878yz.hdf/source = instance('file:///tmp/tmp0ed878yz.hdf/source:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
+link(file:///tmp/tmp0ed878yz.hdf/alocation, file:///tmp/tmp0ed878yz.hdf/alocation, 'bald__array', 'bottom');
+link(file:///tmp/tmp0ed878yz.hdf/alocation, file:///tmp/tmp0ed878yz.hdf/locref, 'bald__references');
+link(file:///tmp/tmp0ed878yz.hdf/alocation, file:///tmp/tmp0ed878yz.hdf/locref2, 'bald__references');
+link(file:///tmp/tmp0ed878yz.hdf/anotherpair, file:///tmp/tmp0ed878yz.hdf/anotherpair, 'bald__array', 'bottom');
+link(file:///tmp/tmp0ed878yz.hdf/apair, file:///tmp/tmp0ed878yz.hdf/anotherpair, 'bald__references');
+link(file:///tmp/tmp0ed878yz.hdf/data, file:///tmp/tmp0ed878yz.hdf/alocation, 'bald__references');
+link(file:///tmp/tmp0ed878yz.hdf/discovery, file:///tmp/tmp0ed878yz.hdf/anotherpair, 'bald__contains', 'top', true);
+link(file:///tmp/tmp0ed878yz.hdf/discovery, file:///tmp/tmp0ed878yz.hdf/apair, 'bald__contains', 'top', true);
+link(file:///tmp/tmp0ed878yz.hdf/discovery, file:///tmp/tmp0ed878yz.hdf/source, 'bald__contains', 'top', true);
+link(file:///tmp/tmp0ed878yz.hdf/referencing, file:///tmp/tmp0ed878yz.hdf/locref, 'bald__contains', 'top', true);
+link(file:///tmp/tmp0ed878yz.hdf/referencing, file:///tmp/tmp0ed878yz.hdf/locref2, 'bald__contains', 'top', true);
+link(file:///tmp/tmp0ed878yz.hdf/root, file:///tmp/tmp0ed878yz.hdf/alocation, 'bald__contains', 'top', true);
+link(file:///tmp/tmp0ed878yz.hdf/root, file:///tmp/tmp0ed878yz.hdf/data, 'bald__contains', 'top', true);
+link(file:///tmp/tmp0ed878yz.hdf/root, file:///tmp/tmp0ed878yz.hdf/discovery, 'bald__contains', 'top', true);
+link(file:///tmp/tmp0ed878yz.hdf/root, file:///tmp/tmp0ed878yz.hdf/referencing, 'bald__contains', 'top', true);
+link(file:///tmp/tmp0ed878yz.hdf/source, file:///tmp/tmp0ed878yz.hdf/institution, 'bald__contains', 'top', true);
     joint.layout.DirectedGraph.layout(graph, { setLinkVertices: false,
                                                nodeSep: 150, rankSep: 100,
                                                marginX: 100, marginY: 100,