Skip to content

Commit a35c9fa

Browse files
marqhmarqh
committed
coord variable first last
1 parent 002e0f8 commit a35c9fa

File tree

4 files changed

+60
-44
lines changed

4 files changed

+60
-44
lines changed

‎lib/bald/__init__.py

Lines changed: 27 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import contextlib
22
import copy
33
import re
4+
import time
45

56
import h5py
67
import jinja2
@@ -236,19 +237,18 @@ def __getitem__(self, item):
236237
if not self.is_http_uri(item):
237238
raise ValueError('{} is not a HTTP URI.'.format(item))
238239
if item not in self.cache:
239-
# import datetime
240-
# now = datetime.datetime.utcnow()
241-
# print('\ndownloading: {}'.format(item))
242-
self.cache[item] = requests.get(item)
240+
# now = time.time()
243241
try:
244-
# Attempt content negotiation, but pass if problems occur.
242+
# print('trying: {}'.format(item))
243+
245244
headers = {'Accept': 'application/rdf+xml'}
246-
self.cache[item] = requests.get(item, headers=headers)
245+
self.cache[item] = requests.get(item, headers=headers, timeout=7)
247246
except Exception:
248-
pass
249-
# then = datetime.datetime.utcnow()
250-
# print('{}s'.format((then-now).total_seconds()))
247+
# print('retrying: {}'.format(item))
248+
headers = {'Accept': 'text/html'}
249+
self.cache[item] = requests.get(item, headers=headers, timeout=7)
251250

251+
# print('in {} seconds'.format(time.time() - then))
252252
return self.cache[item]
253253

254254
def check_uri(self, uri):
@@ -447,7 +447,8 @@ def _graph_elem_attrs(self, remaining_attrs):
447447
else:
448448
kstr = '{key}: '.format(key=attr)
449449
vals = remaining_attrs[attr]
450-
if isinstance(vals, six.string_types):
450+
if (isinstance(vals, six.string_types) or
451+
isinstance(vals, np.ma.core.MaskedConstant)):
451452
vuri = self.unpack_rdfobject(vals, predicate=attr_uri)
452453
if is_http_uri(vuri):
453454
vstr = self.link_template
@@ -748,17 +749,17 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None):
748749

749750
# netCDF coordinate variable special case
750751
if (len(fhandle.variables[name].dimensions) == 1 and
751-
fhandle.variables[name].dimensions[0] == name):
752+
fhandle.variables[name].dimensions[0] == name and
753+
len(fhandle.variables[name]) > 0):
752754
sattrs['bald__array'] = name
753755
sattrs['rdf__type'] = 'bald__Reference'
754756

755-
# if fhandle.variables
756757
sattrs['bald__first_value'] = fhandle.variables[name][0]
757758
if len(fhandle.variables[name]) > 1:
758759
sattrs['bald__last_value'] = fhandle.variables[name][-1]
759760

760761
# datetime special case
761-
if 'units' in fhandle.variables[name].ncattrs():# and False:
762+
if 'units' in fhandle.variables[name].ncattrs():
762763
ustr = fhandle.variables[name].getncattr('units')
763764
pattern = '^([a-z]+) since ([0-9T:\\. -]+)'
764765

@@ -769,15 +770,26 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None):
769770
ig = terra.datetime.ISOGregorian()
770771
tog = terra.datetime.parse_datetime(origin,
771772
calendar=ig)
772-
first = np.array((fhandle.variables[name][0],))
773+
dtype = '{}{}'.format(fhandle.variables[name].dtype.kind,
774+
fhandle.variables[name].dtype.itemsize)
775+
fv = netCDF4.default_fillvals.get(dtype)
776+
if fhandle.variables[name][0] == fv:
777+
first = np.ma.MaskedArray(fhandle.variables[name][0],
778+
mask=True)
779+
else:
780+
first = fhandle.variables[name][0]
773781

774782
edate_first = terra.datetime.EpochDateTimes(first,
775783
quantity,
776784
epoch=tog)
777785

778786
sattrs['bald__first_value'] = str(edate_first)
779787
if len(fhandle.variables[name]) > 1:
780-
last = np.array((fhandle.variables[name][-1],))
788+
if fhandle.variables[name][0] == fv:
789+
last = np.ma.MaskedArray(fhandle.variables[name][-1],
790+
mask=True)
791+
else:
792+
last = fhandle.variables[name][-1]
781793
edate_last = terra.datetime.EpochDateTimes(last,
782794
quantity,
783795
epoch=tog)

‎lib/bald/tests/integration/HTML/hdf_container_nest.html

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -174,33 +174,33 @@
174174
return cell;
175175
};
176176

177-
var file:///tmp/tmp0ed878yz.hdf/alocation = instance('file:///tmp/tmp0ed878yz.hdf/alocation:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>, <a xlink:href="http://binary-array-ld.net/latest/Reference" xlink:show=new text-decoration="underline">bald__Reference</a>', ['<a xlink:href="http://binary-array-ld.net/latest/array" xlink:show=new text-decoration="underline">bald__array</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 11, 17'], '#878800');
178-
var file:///tmp/tmp0ed878yz.hdf/anotherpair = instance('file:///tmp/tmp0ed878yz.hdf/anotherpair:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>, <a xlink:href="http://binary-array-ld.net/latest/Reference" xlink:show=new text-decoration="underline">bald__Reference</a>', ['<a xlink:href="http://binary-array-ld.net/latest/array" xlink:show=new text-decoration="underline">bald__array</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 2'], '#878800');
179-
var file:///tmp/tmp0ed878yz.hdf/apair = instance('file:///tmp/tmp0ed878yz.hdf/apair:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 2'], '#878800');
180-
var file:///tmp/tmp0ed878yz.hdf/data = instance('file:///tmp/tmp0ed878yz.hdf/data:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 11, 17'], '#878800');
181-
var file:///tmp/tmp0ed878yz.hdf/discovery = instance('file:///tmp/tmp0ed878yz.hdf/discovery:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
182-
var file:///tmp/tmp0ed878yz.hdf/institution = instance('file:///tmp/tmp0ed878yz.hdf/institution:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: a quality establishment'], '#878800');
183-
var file:///tmp/tmp0ed878yz.hdf/locref = instance('file:///tmp/tmp0ed878yz.hdf/locref:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: for locational purposes'], '#878800');
184-
var file:///tmp/tmp0ed878yz.hdf/locref2 = instance('file:///tmp/tmp0ed878yz.hdf/locref2:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: for more locational purposes'], '#878800');
185-
var file:///tmp/tmp0ed878yz.hdf/referencing = instance('file:///tmp/tmp0ed878yz.hdf/referencing:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
186-
var file:///tmp/tmp0ed878yz.hdf/root = instance('file:///tmp/tmp0ed878yz.hdf/root:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
187-
var file:///tmp/tmp0ed878yz.hdf/source = instance('file:///tmp/tmp0ed878yz.hdf/source:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
188-
link(file:///tmp/tmp0ed878yz.hdf/alocation, file:///tmp/tmp0ed878yz.hdf/alocation, 'bald__array', 'bottom');
189-
link(file:///tmp/tmp0ed878yz.hdf/alocation, file:///tmp/tmp0ed878yz.hdf/locref, 'bald__references');
190-
link(file:///tmp/tmp0ed878yz.hdf/alocation, file:///tmp/tmp0ed878yz.hdf/locref2, 'bald__references');
191-
link(file:///tmp/tmp0ed878yz.hdf/anotherpair, file:///tmp/tmp0ed878yz.hdf/anotherpair, 'bald__array', 'bottom');
192-
link(file:///tmp/tmp0ed878yz.hdf/apair, file:///tmp/tmp0ed878yz.hdf/anotherpair, 'bald__references');
193-
link(file:///tmp/tmp0ed878yz.hdf/data, file:///tmp/tmp0ed878yz.hdf/alocation, 'bald__references');
194-
link(file:///tmp/tmp0ed878yz.hdf/discovery, file:///tmp/tmp0ed878yz.hdf/anotherpair, 'bald__contains', 'top', true);
195-
link(file:///tmp/tmp0ed878yz.hdf/discovery, file:///tmp/tmp0ed878yz.hdf/apair, 'bald__contains', 'top', true);
196-
link(file:///tmp/tmp0ed878yz.hdf/discovery, file:///tmp/tmp0ed878yz.hdf/source, 'bald__contains', 'top', true);
197-
link(file:///tmp/tmp0ed878yz.hdf/referencing, file:///tmp/tmp0ed878yz.hdf/locref, 'bald__contains', 'top', true);
198-
link(file:///tmp/tmp0ed878yz.hdf/referencing, file:///tmp/tmp0ed878yz.hdf/locref2, 'bald__contains', 'top', true);
199-
link(file:///tmp/tmp0ed878yz.hdf/root, file:///tmp/tmp0ed878yz.hdf/alocation, 'bald__contains', 'top', true);
200-
link(file:///tmp/tmp0ed878yz.hdf/root, file:///tmp/tmp0ed878yz.hdf/data, 'bald__contains', 'top', true);
201-
link(file:///tmp/tmp0ed878yz.hdf/root, file:///tmp/tmp0ed878yz.hdf/discovery, 'bald__contains', 'top', true);
202-
link(file:///tmp/tmp0ed878yz.hdf/root, file:///tmp/tmp0ed878yz.hdf/referencing, 'bald__contains', 'top', true);
203-
link(file:///tmp/tmp0ed878yz.hdf/source, file:///tmp/tmp0ed878yz.hdf/institution, 'bald__contains', 'top', true);
177+
var file:///tmp/tmplkpt7pl7.hdf/alocation = instance('file:///tmp/tmplkpt7pl7.hdf/alocation:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>, <a xlink:href="http://binary-array-ld.net/latest/Reference" xlink:show=new text-decoration="underline">bald__Reference</a>', ['<a xlink:href="http://binary-array-ld.net/latest/array" xlink:show=new text-decoration="underline">bald__array</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 11, 17'], '#878800');
178+
var file:///tmp/tmplkpt7pl7.hdf/anotherpair = instance('file:///tmp/tmplkpt7pl7.hdf/anotherpair:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>, <a xlink:href="http://binary-array-ld.net/latest/Reference" xlink:show=new text-decoration="underline">bald__Reference</a>', ['<a xlink:href="http://binary-array-ld.net/latest/array" xlink:show=new text-decoration="underline">bald__array</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 2'], '#878800');
179+
var file:///tmp/tmplkpt7pl7.hdf/apair = instance('file:///tmp/tmplkpt7pl7.hdf/apair:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 2'], '#878800');
180+
var file:///tmp/tmplkpt7pl7.hdf/data = instance('file:///tmp/tmplkpt7pl7.hdf/data:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/references" xlink:show=new text-decoration="underline">bald__references</a>: |', '<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: 11, 17'], '#878800');
181+
var file:///tmp/tmplkpt7pl7.hdf/discovery = instance('file:///tmp/tmplkpt7pl7.hdf/discovery:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
182+
var file:///tmp/tmplkpt7pl7.hdf/institution = instance('file:///tmp/tmplkpt7pl7.hdf/institution:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: a quality establishment'], '#878800');
183+
var file:///tmp/tmplkpt7pl7.hdf/locref = instance('file:///tmp/tmplkpt7pl7.hdf/locref:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: for locational purposes'], '#878800');
184+
var file:///tmp/tmplkpt7pl7.hdf/locref2 = instance('file:///tmp/tmplkpt7pl7.hdf/locref2:<a xlink:href="http://binary-array-ld.net/latest/Array" xlink:show=new text-decoration="underline">bald__Array</a>', ['<a xlink:href="http://binary-array-ld.net/latest/shape" xlink:show=new text-decoration="underline">bald__shape</a>: |', '<a xlink:href="http://www.w3.org/2004/02/skos/core#prefLabel" xlink:show=new text-decoration="underline">skos__prefLabel</a>: for more locational purposes'], '#878800');
185+
var file:///tmp/tmplkpt7pl7.hdf/referencing = instance('file:///tmp/tmplkpt7pl7.hdf/referencing:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
186+
var file:///tmp/tmplkpt7pl7.hdf/root = instance('file:///tmp/tmplkpt7pl7.hdf/root:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
187+
var file:///tmp/tmplkpt7pl7.hdf/source = instance('file:///tmp/tmplkpt7pl7.hdf/source:<a xlink:href="http://binary-array-ld.net/latest/Container" xlink:show=new text-decoration="underline">bald__Container</a>', ['<a xlink:href="http://binary-array-ld.net/latest/contains" xlink:show=new text-decoration="underline">bald__contains</a>: |'], '#878800');
188+
link(file:///tmp/tmplkpt7pl7.hdf/alocation, file:///tmp/tmplkpt7pl7.hdf/alocation, 'bald__array', 'bottom');
189+
link(file:///tmp/tmplkpt7pl7.hdf/alocation, file:///tmp/tmplkpt7pl7.hdf/locref, 'bald__references');
190+
link(file:///tmp/tmplkpt7pl7.hdf/alocation, file:///tmp/tmplkpt7pl7.hdf/locref2, 'bald__references');
191+
link(file:///tmp/tmplkpt7pl7.hdf/anotherpair, file:///tmp/tmplkpt7pl7.hdf/anotherpair, 'bald__array', 'bottom');
192+
link(file:///tmp/tmplkpt7pl7.hdf/apair, file:///tmp/tmplkpt7pl7.hdf/anotherpair, 'bald__references');
193+
link(file:///tmp/tmplkpt7pl7.hdf/data, file:///tmp/tmplkpt7pl7.hdf/alocation, 'bald__references');
194+
link(file:///tmp/tmplkpt7pl7.hdf/discovery, file:///tmp/tmplkpt7pl7.hdf/anotherpair, 'bald__contains', 'top', true);
195+
link(file:///tmp/tmplkpt7pl7.hdf/discovery, file:///tmp/tmplkpt7pl7.hdf/apair, 'bald__contains', 'top', true);
196+
link(file:///tmp/tmplkpt7pl7.hdf/discovery, file:///tmp/tmplkpt7pl7.hdf/source, 'bald__contains', 'top', true);
197+
link(file:///tmp/tmplkpt7pl7.hdf/referencing, file:///tmp/tmplkpt7pl7.hdf/locref, 'bald__contains', 'top', true);
198+
link(file:///tmp/tmplkpt7pl7.hdf/referencing, file:///tmp/tmplkpt7pl7.hdf/locref2, 'bald__contains', 'top', true);
199+
link(file:///tmp/tmplkpt7pl7.hdf/root, file:///tmp/tmplkpt7pl7.hdf/alocation, 'bald__contains', 'top', true);
200+
link(file:///tmp/tmplkpt7pl7.hdf/root, file:///tmp/tmplkpt7pl7.hdf/data, 'bald__contains', 'top', true);
201+
link(file:///tmp/tmplkpt7pl7.hdf/root, file:///tmp/tmplkpt7pl7.hdf/discovery, 'bald__contains', 'top', true);
202+
link(file:///tmp/tmplkpt7pl7.hdf/root, file:///tmp/tmplkpt7pl7.hdf/referencing, 'bald__contains', 'top', true);
203+
link(file:///tmp/tmplkpt7pl7.hdf/source, file:///tmp/tmplkpt7pl7.hdf/institution, 'bald__contains', 'top', true);
204204
joint.layout.DirectedGraph.layout(graph, { setLinkVertices: false,
205205
nodeSep: 150, rankSep: 100,
206206
marginX: 100, marginY: 100,

0 commit comments

Comments
 (0)