Skip to content

Commit e3ce389

Browse files
jyucsirojyucsiro
committed
Merge branch 'fix-nbarray-issue' into load-tests-triple-store
2 parents 09ab933 + f7f9dfa commit e3ce389

12 files changed

+358
-36
lines changed

‎lib/bald/__init__.py

Lines changed: 59 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,7 @@ def __getitem__(self, item):
240240

241241
def check_uri(self, uri):
242242
result = False
243+
#print("Checking uri: " + uri)
243244
if self[uri].status_code == 200:
244245
result = True
245246
return result
@@ -414,6 +415,12 @@ def rdfnode(self, graph):
414415
selfnode = rdflib.URIRef(self.identity)
415416
for attr in self.attrs:
416417
objs = self.attrs[attr]
418+
if(isinstance(objs, np.ndarray)):
419+
#print("Found np.ndarray")
420+
#print(objs)
421+
#print(attr)
422+
#try to convert np.ndarray to a list
423+
objs = objs.tolist()
417424
if not (isinstance(objs, set) or isinstance(objs, list)):
418425
objs = set([objs])
419426
for obj in objs:
@@ -438,6 +445,11 @@ def rdfgraph(self):
438445
"""
439446
graph = rdflib.Graph()
440447
graph.bind('bald', 'http://binary-array-ld.net/latest/')
448+
for prefix_name in self._prefixes:
449+
#strip the double underscore suffix
450+
new_name = prefix_name[:-2]
451+
452+
graph.bind(new_name, self._prefixes[prefix_name])
441453
graph = self.rdfnode(graph)
442454

443455
return graph
@@ -530,35 +542,53 @@ def load(afilepath):
530542
finally:
531543
f.close()
532544

533-
def load_netcdf(afilepath, uri=None):
545+
def load_netcdf(afilepath, uri=None, baseuri=None):
534546
"""
535547
Validate a file with respect to binary-array-linked-data.
536548
Returns a :class:`bald.validation.Validation`
537549
"""
538550

539551
with load(afilepath) as fhandle:
540-
prefix_group = (fhandle[fhandle.bald__isPrefixedBy] if
552+
prefix_var_name = None
553+
if hasattr(fhandle, 'bald__isPrefixedBy'):
554+
prefix_var_name = fhandle.bald__isPrefixedBy
555+
556+
prefix_var = (fhandle[fhandle.bald__isPrefixedBy] if
541557
hasattr(fhandle, 'bald__isPrefixedBy') else {})
542558
prefixes = {}
543559

544560
skipped_variables = []
545-
if prefix_group != {}:
546-
prefixes = (dict([(prefix, getattr(prefix_group, prefix)) for
547-
prefix in prefix_group.ncattrs()]))
548-
if isinstance(prefix_group, netCDF4._netCDF4.Variable):
549-
skipped_variables.append(prefix_group.name)
561+
if prefix_var != {}:
562+
prefixes = (dict([(prefix, getattr(prefix_var, prefix)) for
563+
prefix in prefix_var.ncattrs()]))
564+
if isinstance(prefix_var, netCDF4._netCDF4.Variable):
565+
skipped_variables.append(prefix_var.name)
550566
else:
551567
for k in fhandle.ncattrs():
552568
if k.endswith('__'):
553569
prefixes[k] = getattr(fhandle, k)
554-
alias_group = (fhandle[fhandle.bald__isAliasedBy]
570+
571+
# check that default set is handled, i.e. bald__ and rdf__
572+
if 'bald__' not in prefixes:
573+
prefixes['bald__'] = "http://binary-array-ld.net/latest/"
574+
575+
if 'rdf__' not in prefixes:
576+
prefixes['rdf__'] = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
577+
578+
#print(prefixes)
579+
580+
alias_var_name = None
581+
if hasattr(fhandle, 'bald__isAliasedBy'):
582+
alias_var_name = fhandle.bald__isAliasedBy
583+
584+
alias_var = (fhandle[fhandle.bald__isAliasedBy]
555585
if hasattr(fhandle, 'bald__isAliasedBy') else {})
556586
aliases = {}
557-
if alias_group != {}:
558-
aliases = (dict([(alias, getattr(alias_group, alias))
559-
for alias in alias_group.ncattrs()]))
560-
if isinstance(alias_group, netCDF4._netCDF4.Variable):
561-
skipped_variables.append(alias_group.name)
587+
if alias_var != {}:
588+
aliases = (dict([(alias, getattr(alias_var, alias))
589+
for alias in alias_var.ncattrs()]))
590+
if isinstance(alias_var, netCDF4._netCDF4.Variable):
591+
skipped_variables.append(alias_var.name)
562592

563593
attrs = {}
564594
for k in fhandle.ncattrs():
@@ -573,32 +603,40 @@ def load_netcdf(afilepath, uri=None):
573603
root_container.attrs['bald__contains'] = []
574604
file_variables = {}
575605
for name in fhandle.variables:
606+
#print(name)
607+
if name == prefix_var_name or name == alias_var_name:
608+
#print("Skipping " + name)
609+
continue
576610

577611
sattrs = fhandle.variables[name].__dict__.copy()
578612
# inconsistent use of '/'; fix it
579613
identity = name
614+
if baseuri is not None:
615+
identity = baseuri + "/" + name
580616

581617
# netCDF coordinate variable special case
582618
if (len(fhandle.variables[name].dimensions) == 1 and
583619
fhandle.variables[name].dimensions[0] == name):
584-
sattrs['bald__array'] = name
620+
#sattrs['bald__array'] = name
621+
sattrs['bald__array'] = identity
585622
sattrs['rdf__type'] = 'bald__Reference'
623+
586624
if fhandle.variables[name].shape:
587625
sattrs['bald__shape'] = fhandle.variables[name].shape
588626
var = Array(identity, sattrs, prefixes=prefixes, aliases=aliases)
589627
else:
590628
var = Subject(identity, sattrs, prefixes=prefixes, aliases=aliases)
591-
if name not in skipped_variables:
592-
# Don't include skipped variables, such as prefix or alias
593-
# variables, within the containment relation.
594-
root_container.attrs['bald__contains'].append(var)
595-
629+
root_container.attrs['bald__contains'].append(var)
596630
file_variables[name] = var
597631

598632

599633

600634
# cycle again and find references
601635
for name in fhandle.variables:
636+
if name == prefix_var_name or name == alias_var_name:
637+
#print("Skipping " + name)
638+
continue
639+
602640
var = file_variables[name]
603641
# reverse lookup based on type to be added
604642
lookups = ['bald__references', 'bald__array']
@@ -624,6 +662,8 @@ def load_netcdf(afilepath, uri=None):
624662
# Else, define a bald:childBroadcast
625663
else:
626664
identity = '{}_{}_ref'.format(name, dim)
665+
if baseuri is not None:
666+
identity = baseuri + '/' + '{}_{}_ref'.format(name, dim)
627667
rattrs = {}
628668
rattrs['rdf__type'] = 'bald__Reference'
629669
reshape = [1 for adim in var_shape]

‎lib/bald/tests/integration/CDL/array_alias_v2.cdl

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
netcdf tmpMwXy8U {
2+
dimensions:
3+
pdim0 = 11 ;
4+
pdim1 = 17 ;
5+
variables:
6+
int prefix_list ;
7+
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
8+
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
9+
10+
int alias_list ;
11+
alias_list:SDN_ParameterDiscoveryCode = "http://vocab.nerc.ac.uk/isoCodelists/sdnCodelists/cdicsrCodeList.xml#SDN_ParameterDiscoveryCode" ;
12+
alias_list:BactTaxaAbundSed = "http://vocab.nerc.ac.uk/collection/P02/current/BAUC/" ;
13+
alias_list:standard_name = "http://def.scitools.org.uk/CFTerms/standard_name" ;
14+
alias_list:air_temperature = "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/" ;
15+
16+
int parent_variable(pdim0, pdim1) ;
17+
parent_variable:rdf__type = "bald__Array" ;
18+
parent_variable:SDN_ParameterDiscoveryCode = "BactTaxaAbundSed" ;
19+
parent_variable:submursible_name = "Nautilus" ;
20+
21+
int temp(pdim0, pdim1) ;
22+
temp:standard_name = "air_temperature" ;
23+
24+
// global attributes:
25+
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
26+
:rdf__type = "bald__Container" ;
27+
:bald__isPrefixedBy = "prefix_list" ;
28+
:bald__isAliasedBy = "alias_list" ;
29+
data:
30+
31+
parent_variable =
32+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
33+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
34+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
35+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
36+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
37+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
38+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
39+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
40+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
41+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
42+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;
43+
44+
temp =
45+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
46+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
47+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
48+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
49+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
50+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
51+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
52+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
53+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
54+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
55+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;
56+
57+
58+
}

‎lib/bald/tests/integration/CDL/array_geo.cdl

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
netcdf tmpMwXy8U {
2+
dimensions:
3+
pdim0 = 11 ;
4+
pdim1 = 17 ;
5+
variables:
6+
int prefix_list ;
7+
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
8+
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
9+
prefix_list:rdfs__ = "http://www.w3.org/2000/01/rdf-schema#" ;
10+
prefix_list:cf__ = "http://def.scitools.org.uk/CFTerms/" ;
11+
prefix_list:geo__ = "http://www.opengis.net/ont/geosparql#" ;
12+
13+
int temp(pdim0, pdim1) ;
14+
temp:cf__standard_name = "air_temperature" ;
15+
temp:cf__long_name = "Air temperature obs example at point" ;
16+
temp:rdfs__label = "Air temperature obs example at point" ;
17+
temp:geo__asWKT = "POINT(-77.03524 38.889468)" ;
18+
19+
int pressure(pdim0, pdim1) ;
20+
pressure:cf__standard_name = "air_pressure" ;
21+
pressure:cf__long_name = "Air pressure at UCAR Centre Green" ;
22+
pressure:rdfs__label = "Air pressure at UCAR Centre Green" ;
23+
pressure:geo__asWKT = "POINT(-105.24584700000003 40.0315278)" ;
24+
25+
// global attributes:
26+
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
27+
:rdf__type = "bald__Container" ;
28+
:bald__isPrefixedBy = "prefix_list" ;
29+
data:
30+
31+
temp =
32+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
33+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
34+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
35+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
36+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
37+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
38+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
39+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
40+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
41+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
42+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;
43+
44+
45+
}

‎lib/bald/tests/integration/CDL/array_prefix_v2.cdl

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
netcdf tmpMwXy8U {
2+
dimensions:
3+
pdim0 = 11 ;
4+
pdim1 = 17 ;
5+
variables:
6+
int prefix_list ;
7+
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
8+
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
9+
prefix_list:sdn__ = "http://vocab.nerc.ac.uk/isoCodelists/sdnCodelists/cdicsrCodeList.xml#" ;
10+
prefix_list:sdn-vocab__= "http://vocab.nerc.ac.uk/collection/P02/current/" ;
11+
prefix_list:cf__ = "http://def.scitools.org.uk/CFTerms/" ;
12+
prefix_list:cfsn__ = "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/" ;
13+
14+
int parent_variable(pdim0, pdim1) ;
15+
parent_variable:rdf__type = "bald__Array" ;
16+
parent_variable:sdn__SDN_ParameterDiscoveryCode = "sdn-vocab__BAUC" ;
17+
parent_variable:submursible_name = "Nautilus" ;
18+
19+
int temp(pdim0, pdim1) ;
20+
temp:standard_name = "air_temperature" ;
21+
22+
// global attributes:
23+
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
24+
:rdf__type = "bald__Container" ;
25+
:bald__isPrefixedBy = "prefix_list" ;
26+
data:
27+
28+
parent_variable =
29+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
30+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
31+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
32+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
33+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
34+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
35+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
36+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
37+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
38+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
39+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;
40+
41+
temp =
42+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
43+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
44+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
45+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
46+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
47+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
48+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
49+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
50+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
51+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
52+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;
53+
54+
55+
}

‎lib/bald/tests/integration/CDL/array_prefix_v2_full.cdl

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
netcdf tmpMwXy8U {
2+
dimensions:
3+
pdim0 = 11 ;
4+
pdim1 = 17 ;
5+
variables:
6+
int prefix_list ;
7+
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
8+
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
9+
prefix_list:sdn__ = "http://vocab.nerc.ac.uk/isoCodelists/sdnCodelists/cdicsrCodeList.xml#" ;
10+
prefix_list:sdn-vocab__= "http://vocab.nerc.ac.uk/collection/P02/current/" ;
11+
prefix_list:cf__ = "http://def.scitools.org.uk/CFTerms/" ;
12+
prefix_list:cfsn-mmi__ = "http://mmisw.org/ont/cf/parameter/" ;
13+
prefix_list:cfsn-nerc__ = "http://vocab.nerc.ac.uk/collection/P07/current/";
14+
15+
int parent_variable(pdim0, pdim1) ;
16+
parent_variable:rdf__type = "bald__Array" ;
17+
parent_variable:sdn__SDN_ParameterDiscoveryCode = "sdn-vocab__BAUC" ;
18+
parent_variable:submursible_name = "Nautilus" ;
19+
20+
int temp(pdim0, pdim1) ;
21+
temp:cf__standard_name = "cfsn-mmi__air_temperature" ;
22+
//temp:cf__standard_name = "cfsn-nerc__CFSN0023";
23+
24+
// global attributes:
25+
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
26+
:rdf__type = "bald__Container" ;
27+
:bald__isPrefixedBy = "prefix_list" ;
28+
data:
29+
30+
parent_variable =
31+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
32+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
33+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
34+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
35+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
36+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
37+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
38+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
39+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
40+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
41+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;
42+
43+
temp =
44+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
45+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
46+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
47+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
48+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
49+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
50+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
51+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
52+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
53+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
54+
_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;
55+
56+
57+
}

‎lib/bald/tests/integration/CDL/multi_array_reference.cdl

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@ dimensions:
33
pdim0 = 11 ;
44
pdim1 = 17 ;
55
variables:
6+
int prefix_list(pdim0, pdim1) ;
7+
prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
8+
prefix_list:metce__ = "http://codes.wmo.int/common/observation-type/METCE/2013/" ;
9+
prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
10+
611
int variable1(pdim0, pdim1) ;
712
variable1:bald__references = "location_variable" ;
813
variable1:long_name = "Gerald";
@@ -31,11 +36,4 @@ variables:
3136
:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
3237
:bald__isPrefixedBy = "prefix_list" ;
3338

34-
group: prefix_list {
35-
36-
// group attributes:
37-
:bald__ = "http://binary-array-ld.net/latest/" ;
38-
:metce__ = "http://codes.wmo.int/common/observation-type/METCE/2013/" ;
39-
:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
40-
} // group bald__prefix_list
4139
}

0 commit comments

Comments
 (0)