Skip to content

Commit e0b7bc1

Browse files
committed
use cache
1 parent e187465 commit e0b7bc1

11 files changed

+84
-79
lines changed

lib/bald/__init__.py

Lines changed: 25 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -235,13 +235,13 @@ def __getitem__(self, item):
235235
if not self.is_http_uri(item):
236236
raise ValueError('{} is not a HTTP URI.'.format(item))
237237
if item not in self.cache:
238-
headers = {'Accept': 'text/turtle'}
239-
# import datetime
240-
# now = datetime.datetime.utcnow()
241-
# print('\ndownloading: {}'.format(item))
238+
headers = {'Accept': 'application/rdf+xml'}
239+
import datetime
240+
now = datetime.datetime.utcnow()
241+
print('\ndownloading: {}'.format(item))
242242
self.cache[item] = requests.get(item, headers=headers)
243-
# then = datetime.datetime.utcnow()
244-
# print('{}s'.format((then-now).total_seconds()))
243+
then = datetime.datetime.utcnow()
244+
print('{}s'.format((then-now).total_seconds()))
245245

246246
return self.cache[item]
247247

@@ -387,9 +387,6 @@ def unpack_rdfobject(self, astring, predicate):
387387
# qres = self.alias_graph.query(rdfobj_alias_query)
388388
try:
389389
qres = self.alias_graph.query(rdfobj_alias_query)
390-
# except Exception:
391-
# import pdb; pdb.set_trace()
392-
# qres = self.alias_graph.query(rdfobj_alias_query)
393390
results = list(qres)
394391
if len(results) > 1:
395392
raise ValueError('multiple alias options')
@@ -637,13 +634,16 @@ def load(afilepath):
637634
finally:
638635
f.close()
639636

640-
def load_netcdf(afilepath, baseuri=None, alias_dict=None):
637+
def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None):
641638
"""
642639
Load a file with respect to binary-array-linked-data.
643640
Returns a :class:`bald.Collection`
644641
"""
645642
if alias_dict == None:
646643
alias_dict = {}
644+
if cache is None:
645+
cache = HttpCache()
646+
647647
with load(afilepath) as fhandle:
648648
if baseuri is None:
649649
baseuri = 'file://{}'.format(afilepath)
@@ -693,9 +693,11 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None):
693693
attrs[k] = getattr(fhandle, k)
694694

695695
aliasgraph = rdflib.Graph()
696+
696697
for alias in aliases:
698+
response = cache[aliases[alias]]
697699
try:
698-
aliasgraph.parse(aliases[alias], format='xml')
700+
aliasgraph.parse(data=response.text, format='xml')
699701
except TypeError:
700702
pass
701703
# if hasattr(fhandle, 'Conventions'):
@@ -715,7 +717,6 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None):
715717
# if len(set(na_keys)) != len(na_keys):
716718
# raise ValueError('duplicate aliases')
717719
# aliases = careful_update(aliases, dict(new_aliases))
718-
719720
root_container = Container(baseuri, '', attrs, prefixes=prefixes,
720721
aliases=aliases, alias_graph=aliasgraph)
721722

@@ -842,24 +843,24 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None):
842843
return root_container
843844

844845

845-
def validate_netcdf(afilepath, cache=None, baseuri=None):
846+
def validate_netcdf(afilepath, baseuri=None, cache=None):
846847
"""
847848
Validate a file with respect to binary-array-linked-data.
848849
Returns a :class:`bald.validation.Validation`
849850
850851
"""
851-
root_container = load_netcdf(afilepath, baseuri=baseuri)
852+
root_container = load_netcdf(afilepath, baseuri=baseuri, cache=cache)
852853
return validate(root_container, cache=cache)
853854

854855

855-
def validate_hdf5(afilepath, cache=None, baseuri=None):
856+
def validate_hdf5(afilepath, baseuri=None, cache=None):
856857
"""
857858
Validate a file with respect to binary-array-linked-data.
858859
Returns a :class:`bald.validation.Validation`
859860
860861
"""
861-
root_container = load_hdf5(afilepath, baseuri=baseuri)
862-
return validate(root_container)
862+
root_container = load_hdf5(afilepath, baseuri=baseuri, cache=cache)
863+
return validate(root_container, cache=cache)
863864

864865
def validate(root_container, sval=None, cache=None):
865866
"""
@@ -896,20 +897,24 @@ def careful_update(adict, bdict):
896897
adict.update(bdict)
897898
return adict
898899

899-
def load_hdf5(afilepath, baseuri=None, alias_dict=None):
900+
def load_hdf5(afilepath, baseuri=None, alias_dict=None, cache=None):
901+
if cache is None:
902+
cache = HttpCache()
900903
with load(afilepath) as fhandle:
901904
# unused?
902905
cache = {}
903906
if baseuri is None:
904907
baseuri = 'file://{}'.format(afilepath)
905908

906909
root_container, file_variables = _hdf_group(fhandle, baseuri=baseuri,
907-
alias_dict=alias_dict)
910+
alias_dict=alias_dict, cache=cache)
908911
_hdf_references(fhandle, root_container, file_variables)
909912
return root_container
910913

911914
def _hdf_group(fhandle, identity='root', baseuri=None, prefixes=None,
912-
aliases=None, alias_dict=None):
915+
aliases=None, alias_dict=None, cache=None):
916+
if cache is None:
917+
cache = HttpCache()
913918

914919
prefix_group = fhandle.attrs.get('bald__isPrefixedBy')
915920
if prefixes is None:

lib/bald/tests/integration/HTML/hdf_container_nest.html

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -174,33 +174,33 @@
174174
return cell;
175175
};
176176

177-
var file:///tmp/tmpk7nc3lox.hdf/alocation = instance('file:///tmp/tmpk7nc3lox.hdf/alocation:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>, <a xlink:href="http://binary-array-ld.net/latest/Reference" xlink:show=new text-decoration="underline">bald__Reference</a>', ['<a xlink:href="http://binary-array-ld.net/latest/array" xlink:show=new text-decoration="underline">bald__array</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 11, 17'], '#878800');
178-
var file:///tmp/tmpk7nc3lox.hdf/anotherpair = instance('file:///tmp/tmpk7nc3lox.hdf/anotherpair:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>, <a xlink:href="http://binary-array-ld.net/latest/Reference" xlink:show=new text-decoration="underline">bald__Reference</a>', ['<a xlink:href="http://binary-array-ld.net/latest/array" xlink:show=new text-decoration="underline">bald__array</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 2'], '#878800');
179-
var file:///tmp/tmpk7nc3lox.hdf/apair = instance('file:///tmp/tmpk7nc3lox.hdf/apair:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 2'], '#878800');
180-
var file:///tmp/tmpk7nc3lox.hdf/data = instance('file:///tmp/tmpk7nc3lox.hdf/data:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 11, 17'], '#878800');
181-
var file:///tmp/tmpk7nc3lox.hdf/discovery = instance('file:///tmp/tmpk7nc3lox.hdf/discovery:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
182-
var file:///tmp/tmpk7nc3lox.hdf/institution = instance('file:///tmp/tmpk7nc3lox.hdf/institution:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: a quality establishment'], '#878800');
183-
var file:///tmp/tmpk7nc3lox.hdf/locref = instance('file:///tmp/tmpk7nc3lox.hdf/locref:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: for locational purposes'], '#878800');
184-
var file:///tmp/tmpk7nc3lox.hdf/locref2 = instance('file:///tmp/tmpk7nc3lox.hdf/locref2:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: for more locational purposes'], '#878800');
185-
var file:///tmp/tmpk7nc3lox.hdf/referencing = instance('file:///tmp/tmpk7nc3lox.hdf/referencing:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
186-
var file:///tmp/tmpk7nc3lox.hdf/root = instance('file:///tmp/tmpk7nc3lox.hdf/root:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
187-
var file:///tmp/tmpk7nc3lox.hdf/source = instance('file:///tmp/tmpk7nc3lox.hdf/source:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
188-
link(file:///tmp/tmpk7nc3lox.hdf/alocation, file:///tmp/tmpk7nc3lox.hdf/alocation, 'bald__array', 'bottom');
189-
link(file:///tmp/tmpk7nc3lox.hdf/alocation, file:///tmp/tmpk7nc3lox.hdf/locref, 'bald__references');
190-
link(file:///tmp/tmpk7nc3lox.hdf/alocation, file:///tmp/tmpk7nc3lox.hdf/locref2, 'bald__references');
191-
link(file:///tmp/tmpk7nc3lox.hdf/anotherpair, file:///tmp/tmpk7nc3lox.hdf/anotherpair, 'bald__array', 'bottom');
192-
link(file:///tmp/tmpk7nc3lox.hdf/apair, file:///tmp/tmpk7nc3lox.hdf/anotherpair, 'bald__references');
193-
link(file:///tmp/tmpk7nc3lox.hdf/data, file:///tmp/tmpk7nc3lox.hdf/alocation, 'bald__references');
194-
link(file:///tmp/tmpk7nc3lox.hdf/discovery, file:///tmp/tmpk7nc3lox.hdf/anotherpair, 'bald__contains', 'top', true);
195-
link(file:///tmp/tmpk7nc3lox.hdf/discovery, file:///tmp/tmpk7nc3lox.hdf/apair, 'bald__contains', 'top', true);
196-
link(file:///tmp/tmpk7nc3lox.hdf/discovery, file:///tmp/tmpk7nc3lox.hdf/source, 'bald__contains', 'top', true);
197-
link(file:///tmp/tmpk7nc3lox.hdf/referencing, file:///tmp/tmpk7nc3lox.hdf/locref, 'bald__contains', 'top', true);
198-
link(file:///tmp/tmpk7nc3lox.hdf/referencing, file:///tmp/tmpk7nc3lox.hdf/locref2, 'bald__contains', 'top', true);
199-
link(file:///tmp/tmpk7nc3lox.hdf/root, file:///tmp/tmpk7nc3lox.hdf/alocation, 'bald__contains', 'top', true);
200-
link(file:///tmp/tmpk7nc3lox.hdf/root, file:///tmp/tmpk7nc3lox.hdf/data, 'bald__contains', 'top', true);
201-
link(file:///tmp/tmpk7nc3lox.hdf/root, file:///tmp/tmpk7nc3lox.hdf/discovery, 'bald__contains', 'top', true);
202-
link(file:///tmp/tmpk7nc3lox.hdf/root, file:///tmp/tmpk7nc3lox.hdf/referencing, 'bald__contains', 'top', true);
203-
link(file:///tmp/tmpk7nc3lox.hdf/source, file:///tmp/tmpk7nc3lox.hdf/institution, 'bald__contains', 'top', true);
177+
var file:///tmp/tmp9cq2_b_a.hdf/alocation = instance('file:///tmp/tmp9cq2_b_a.hdf/alocation:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>, <a xlink:href="http://binary-array-ld.net/latest/Reference" xlink:show=new text-decoration="underline">bald__Reference</a>', ['<a xlink:href="http://binary-array-ld.net/latest/array" xlink:show=new text-decoration="underline">bald__array</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 11, 17'], '#878800');
178+
var file:///tmp/tmp9cq2_b_a.hdf/anotherpair = instance('file:///tmp/tmp9cq2_b_a.hdf/anotherpair:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>, <a xlink:href="http://binary-array-ld.net/latest/Reference" xlink:show=new text-decoration="underline">bald__Reference</a>', ['<a xlink:href="http://binary-array-ld.net/latest/array" xlink:show=new text-decoration="underline">bald__array</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 2'], '#878800');
179+
var file:///tmp/tmp9cq2_b_a.hdf/apair = instance('file:///tmp/tmp9cq2_b_a.hdf/apair:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 2'], '#878800');
180+
var file:///tmp/tmp9cq2_b_a.hdf/data = instance('file:///tmp/tmp9cq2_b_a.hdf/data:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 11, 17'], '#878800');
181+
var file:///tmp/tmp9cq2_b_a.hdf/discovery = instance('file:///tmp/tmp9cq2_b_a.hdf/discovery:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
182+
var file:///tmp/tmp9cq2_b_a.hdf/institution = instance('file:///tmp/tmp9cq2_b_a.hdf/institution:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: a quality establishment'], '#878800');
183+
var file:///tmp/tmp9cq2_b_a.hdf/locref = instance('file:///tmp/tmp9cq2_b_a.hdf/locref:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: for locational purposes'], '#878800');
184+
var file:///tmp/tmp9cq2_b_a.hdf/locref2 = instance('file:///tmp/tmp9cq2_b_a.hdf/locref2:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: for more locational purposes'], '#878800');
185+
var file:///tmp/tmp9cq2_b_a.hdf/referencing = instance('file:///tmp/tmp9cq2_b_a.hdf/referencing:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
186+
var file:///tmp/tmp9cq2_b_a.hdf/root = instance('file:///tmp/tmp9cq2_b_a.hdf/root:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
187+
var file:///tmp/tmp9cq2_b_a.hdf/source = instance('file:///tmp/tmp9cq2_b_a.hdf/source:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
188+
link(file:///tmp/tmp9cq2_b_a.hdf/alocation, file:///tmp/tmp9cq2_b_a.hdf/alocation, 'bald__array', 'bottom');
189+
link(file:///tmp/tmp9cq2_b_a.hdf/alocation, file:///tmp/tmp9cq2_b_a.hdf/locref, 'bald__references');
190+
link(file:///tmp/tmp9cq2_b_a.hdf/alocation, file:///tmp/tmp9cq2_b_a.hdf/locref2, 'bald__references');
191+
link(file:///tmp/tmp9cq2_b_a.hdf/anotherpair, file:///tmp/tmp9cq2_b_a.hdf/anotherpair, 'bald__array', 'bottom');
192+
link(file:///tmp/tmp9cq2_b_a.hdf/apair, file:///tmp/tmp9cq2_b_a.hdf/anotherpair, 'bald__references');
193+
link(file:///tmp/tmp9cq2_b_a.hdf/data, file:///tmp/tmp9cq2_b_a.hdf/alocation, 'bald__references');
194+
link(file:///tmp/tmp9cq2_b_a.hdf/discovery, file:///tmp/tmp9cq2_b_a.hdf/anotherpair, 'bald__contains', 'top', true);
195+
link(file:///tmp/tmp9cq2_b_a.hdf/discovery, file:///tmp/tmp9cq2_b_a.hdf/apair, 'bald__contains', 'top', true);
196+
link(file:///tmp/tmp9cq2_b_a.hdf/discovery, file:///tmp/tmp9cq2_b_a.hdf/source, 'bald__contains', 'top', true);
197+
link(file:///tmp/tmp9cq2_b_a.hdf/referencing, file:///tmp/tmp9cq2_b_a.hdf/locref, 'bald__contains', 'top', true);
198+
link(file:///tmp/tmp9cq2_b_a.hdf/referencing, file:///tmp/tmp9cq2_b_a.hdf/locref2, 'bald__contains', 'top', true);
199+
link(file:///tmp/tmp9cq2_b_a.hdf/root, file:///tmp/tmp9cq2_b_a.hdf/alocation, 'bald__contains', 'top', true);
200+
link(file:///tmp/tmp9cq2_b_a.hdf/root, file:///tmp/tmp9cq2_b_a.hdf/data, 'bald__contains', 'top', true);
201+
link(file:///tmp/tmp9cq2_b_a.hdf/root, file:///tmp/tmp9cq2_b_a.hdf/discovery, 'bald__contains', 'top', true);
202+
link(file:///tmp/tmp9cq2_b_a.hdf/root, file:///tmp/tmp9cq2_b_a.hdf/referencing, 'bald__contains', 'top', true);
203+
link(file:///tmp/tmp9cq2_b_a.hdf/source, file:///tmp/tmp9cq2_b_a.hdf/institution, 'bald__contains', 'top', true);
204204
joint.layout.DirectedGraph.layout(graph, { setLinkVertices: false,
205205
nodeSep: 150, rankSep: 100,
206206
marginX: 100, marginY: 100,

lib/bald/tests/integration/test_aliases.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ def test_valid_uri(self):
3737
dsetp = f.create_dataset("parent_dataset", (11, 17), dtype='i')
3838
dsetp.attrs['skosPrefLabel'] = 'alabel'
3939
f.close()
40-
validation = bald.validate_hdf5(tfile)
40+
validation = bald.validate_hdf5(tfile, cache=self.acache)
4141
exns = validation.exceptions()
4242
self.assertTrue(validation.is_valid(), msg='{} != []'.format(exns))
4343

@@ -52,7 +52,7 @@ def test_invalid_uri(self):
5252
dsetp = f.create_dataset("parent_dataset", (11, 17), dtype='i')
5353
dsetp.attrs['skosPrefLabel'] = 'alabel'
5454
f.close()
55-
validation = bald.validate_hdf5(tfile)
55+
validation = bald.validate_hdf5(tfile, cache=self.acache)
5656
exns = validation.exceptions()
5757
expected = ['http://binary-array-ld.net/latest/turtle is not resolving as a resource (404).',
5858
'http://binary-array-ld.net/latest/walnut is not resolving as a resource (404).']

0 commit comments

Comments
 (0)