Merge pull request #71 from jyucsiro/load-tests-triple-store

Load tests triple store

Author: marqh

lib/bald/__init__.py

@@ -414,6 +414,9 @@ def rdfnode(self, graph):
         selfnode = rdflib.URIRef(self.identity)
         for attr in self.attrs:
             objs = self.attrs[attr]
+            if(isinstance(objs, np.ndarray)):
+                #try to convert np.ndarray to a list
+                objs = objs.tolist()
             if not (isinstance(objs, set) or isinstance(objs, list)):
                 objs = set([objs])
             for obj in objs:
@@ -438,6 +441,11 @@ def rdfgraph(self):
         """
         graph = rdflib.Graph()
         graph.bind('bald', 'http://binary-array-ld.net/latest/')
+        for prefix_name in self._prefixes:
+           #strip the double underscore suffix
+           new_name = prefix_name[:-2]
+
+           graph.bind(new_name, self._prefixes[prefix_name])
         graph = self.rdfnode(graph)
 
         return graph
@@ -530,75 +538,96 @@ def load(afilepath):
     finally:
         f.close()
 
-def load_netcdf(afilepath, uri=None):
+def load_netcdf(afilepath, baseuri=None):
     """
     Validate a file with respect to binary-array-linked-data.
     Returns a :class:`bald.validation.Validation`
     """
 
     with load(afilepath) as fhandle:
-        prefix_group = (fhandle[fhandle.bald__isPrefixedBy] if
+        prefix_var_name  = None
+        if hasattr(fhandle, 'bald__isPrefixedBy'):
+           prefix_var_name  = fhandle.bald__isPrefixedBy
+
+        prefix_var = (fhandle[fhandle.bald__isPrefixedBy] if
                         hasattr(fhandle, 'bald__isPrefixedBy') else {})
         prefixes = {}
 
         skipped_variables = []
-        if prefix_group != {}:
-            prefixes = (dict([(prefix, getattr(prefix_group, prefix)) for
-                              prefix in prefix_group.ncattrs()]))
-            if isinstance(prefix_group, netCDF4._netCDF4.Variable):
-                skipped_variables.append(prefix_group.name)
+        if prefix_var != {}:
+            prefixes = (dict([(prefix, getattr(prefix_var, prefix)) for
+                              prefix in prefix_var.ncattrs()]))
+            if isinstance(prefix_var, netCDF4._netCDF4.Variable):
+                skipped_variables.append(prefix_var.name)
         else:
             for k in fhandle.ncattrs():
                 if k.endswith('__'):
                     prefixes[k] = getattr(fhandle, k)
-        alias_group = (fhandle[fhandle.bald__isAliasedBy]
+
+        # check that default set is handled, i.e. bald__ and rdf__
+        if 'bald__' not in prefixes:
+            prefixes['bald__'] = "http://binary-array-ld.net/latest/" 
+
+        if 'rdf__' not in prefixes:
+            prefixes['rdf__'] = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+
+        alias_var_name = None
+        if hasattr(fhandle, 'bald__isAliasedBy'):
+           alias_var_name  = fhandle.bald__isAliasedBy
+
+        alias_var = (fhandle[fhandle.bald__isAliasedBy]
                        if hasattr(fhandle, 'bald__isAliasedBy') else {})
         aliases = {}
-        if alias_group != {}:
-            aliases = (dict([(alias, getattr(alias_group, alias))
-                             for alias in alias_group.ncattrs()]))
-            if isinstance(alias_group, netCDF4._netCDF4.Variable):
-                skipped_variables.append(alias_group.name)
+        if alias_var != {}:
+            aliases = (dict([(alias, getattr(alias_var, alias))
+                             for alias in alias_var.ncattrs()]))
+            if isinstance(alias_var, netCDF4._netCDF4.Variable):
+                skipped_variables.append(alias_var.name)
 
         attrs = {}
         for k in fhandle.ncattrs():
             attrs[k] = getattr(fhandle, k)
         # It would be nice to use the URI of the file if it is known.
-        if uri is not None:
-            identity = uri
+        if baseuri is not None:
+            identity = baseuri
         else:
             identity = 'root'
         root_container = Container(identity, attrs, prefixes=prefixes,
                                    aliases=aliases)
         root_container.attrs['bald__contains'] = []
         file_variables = {}
         for name in fhandle.variables:
+            if name ==  prefix_var_name or name == alias_var_name:
+                continue
 
             sattrs = fhandle.variables[name].__dict__.copy()
             # inconsistent use of '/'; fix it
             identity = name
+            if baseuri is not None:
+                identity = baseuri + "/" + name
 
             # netCDF coordinate variable special case
             if (len(fhandle.variables[name].dimensions) == 1 and
                 fhandle.variables[name].dimensions[0] == name):
-                sattrs['bald__array'] = name
+                #sattrs['bald__array'] = name
+                sattrs['bald__array'] = identity
                 sattrs['rdf__type'] = 'bald__Reference'
+                
             if fhandle.variables[name].shape:
                 sattrs['bald__shape'] = fhandle.variables[name].shape
                 var = Array(identity, sattrs, prefixes=prefixes, aliases=aliases)
             else:
                 var = Subject(identity, sattrs, prefixes=prefixes, aliases=aliases)
-            if name not in skipped_variables:
-                # Don't include skipped variables, such as prefix or alias
-                # variables, within the containment relation.
-                root_container.attrs['bald__contains'].append(var)
-
+            root_container.attrs['bald__contains'].append(var)
             file_variables[name] = var
 
 
 
         # cycle again and find references
         for name in fhandle.variables:
+            if name ==  prefix_var_name or name == alias_var_name:
+                continue
+
             var = file_variables[name]
             # reverse lookup based on type to be added
             lookups = ['bald__references', 'bald__array']
@@ -624,6 +653,8 @@ def load_netcdf(afilepath, uri=None):
                         # Else, define a bald:childBroadcast
                         else:
                             identity = '{}_{}_ref'.format(name, dim)
+                            if baseuri is not None:
+                                identity = baseuri + '/' +  '{}_{}_ref'.format(name, dim)
                             rattrs = {}
                             rattrs['rdf__type'] = 'bald__Reference'
                             reshape = [1 for adim in var_shape]

lib/bald/tests/integration/CDL/array_geo.cdl

@@ -0,0 +1,31 @@
+netcdf tmpMwXy8U {
+dimensions:
+	pdim0 = 11 ;
+	pdim1 = 17 ;
+variables:
+	int prefix_list ;
+  		prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
+  		prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
+		prefix_list:rdfs__ = "http://www.w3.org/2000/01/rdf-schema#" ;
+		prefix_list:cf__ = "http://def.scitools.org.uk/CFTerms/" ;
+		prefix_list:geo__ = "http://www.opengis.net/ont/geosparql#" ;
+
+	int temp(pdim0, pdim1) ;
+	        temp:cf__standard_name = "air_temperature" ;
+	        temp:cf__long_name = "Air temperature obs example at point" ;
+	        temp:rdfs__label = "Air temperature obs example at point" ;
+		temp:geo__asWKT = "POINT(-77.03524 38.889468)" ;
+
+	int pressure(pdim0, pdim1) ;
+	        pressure:cf__standard_name = "air_pressure" ;
+	        pressure:cf__long_name = "Air pressure at UCAR Centre Green" ;
+	        pressure:rdfs__label = "Air pressure at UCAR Centre Green" ;
+		pressure:geo__asWKT = "POINT(-105.24584700000003 40.0315278)" ;
+
+// global attributes:
+		:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
+		:rdf__type = "bald__Container" ;
+		:bald__isPrefixedBy = "prefix_list" ;
+data:
+
+}

lib/bald/tests/integration/CDL/array_multitypes.cdl

@@ -0,0 +1,32 @@
+netcdf tmpMwXy8U {
+dimensions:
+	pdim0 = 11 ;
+	pdim1 = 17 ;
+variables:
+	int prefix_list ;
+  		prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
+  		prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
+		prefix_list:rdfs__ = "http://www.w3.org/2000/01/rdf-schema#" ;
+		prefix_list:cf__ = "http://def.scitools.org.uk/CFTerms/" ;
+		prefix_list:geo__ = "http://www.opengis.net/ont/geosparql#" ;
+
+	int temp(pdim0, pdim1) ;
+	        temp:cf__standard_name = "air_temperature" ;
+	        temp:cf__long_name = "Air temperature obs example at point" ;
+	        temp:rdfs__label = "Air temperature obs example at point" ;
+		temp:geo__asWKT = "POINT(-77.03524 38.889468)" ;
+
+	int pressure(pdim0, pdim1) ;
+		:rdf__type = "geo__Geometry" ;
+	        pressure:cf__standard_name = "air_pressure" ;
+	        pressure:cf__long_name = "Air pressure at UCAR Centre Green" ;
+	        pressure:rdfs__label = "Air pressure at UCAR Centre Green" ;
+		pressure:geo__asWKT = "POINT(-105.24584700000003 40.0315278)" ;
+
+// global attributes:
+		:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
+		:rdf__type = "bald__Container" ;
+		:bald__isPrefixedBy = "prefix_list" ;
+data:
+
+}

lib/bald/tests/integration/CDL/multi_array_reference.cdl

@@ -3,6 +3,11 @@ dimensions:
 	pdim0 = 11 ;
 	pdim1 = 17 ;
 variables:
+        int prefix_list(pdim0, pdim1) ;
+  		prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
+		prefix_list:metce__ = "http://codes.wmo.int/common/observation-type/METCE/2013/" ;
+  		prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
+		
 	int variable1(pdim0, pdim1) ;
 		variable1:bald__references = "location_variable" ;
 		variable1:long_name = "Gerald";
@@ -31,11 +36,4 @@ variables:
 		:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
 		:bald__isPrefixedBy = "prefix_list" ;
 
-group: prefix_list {
-
-  // group attributes:
-  		:bald__ = "http://binary-array-ld.net/latest/" ;
-		:metce__ = "http://codes.wmo.int/common/observation-type/METCE/2013/" ;
-  		:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
-  } // group bald__prefix_list
 }

lib/bald/tests/integration/TTL/array_reference_withbase.ttl

@@ -0,0 +1,20 @@
+@prefix bald: <http://binary-array-ld.net/latest/> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix xml: <http://www.w3.org/XML/1998/namespace> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+<http://example.org/base> a bald:Container ;
+    bald:contains <http://example.org/base/child_variable>,
+        <http://example.org/base/parent_variable> ;
+    bald:isPrefixedBy "prefix_list" .
+
+<http://example.org/base/parent_variable> a bald:Array ;
+    bald:references <http://example.org/base/child_variable> ;
+    bald:shape "(11, 17)" .
+
+<http://example.org/base/child_variable> a bald:Array,
+        bald:Reference ;
+    bald:array <http://example.org/base/child_variable> ;
+    bald:shape "(11, 17)" .
+

lib/bald/tests/integration/TTL/multi_array_reference.ttl

@@ -1,4 +1,5 @@
 @prefix bald: <http://binary-array-ld.net/latest/> .
+@prefix metce: <http://codes.wmo.int/common/observation-type/METCE/2013/> .
 @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
 @prefix xml: <http://www.w3.org/XML/1998/namespace> .
@@ -21,15 +22,15 @@
         <variable1_pdim0_ref> ;
     bald:shape "(11, 17)" ;
     <long_name> "Gerald" ;
-    <obtype> <http://codes.wmo.int/common/observation-type/METCE/2013/SamplingObservation> .
+    <obtype> metce:SamplingObservation .
 
 <variable2> a bald:Array ;
     bald:references <location_variable>,
         <pdim1>,
         <variable2_pdim0_ref> ;
     bald:shape "(11, 17)" ;
     <long_name> "Imelda" ;
-    <obtype> <http://codes.wmo.int/common/observation-type/METCE/2013/SamplingObservation> .
+    <obtype> metce:SamplingObservation .
 
 <variable1_pdim0_ref> a bald:Reference,
         bald:Subject ;

lib/bald/tests/integration/test_cdl.py

@@ -64,8 +64,9 @@ def test_grid_OISST_GHRSST(self):
         validation = bald.validate_netcdf(tfile)
         exns = validation.exceptions()
         exns.sort()
-        expected = ['http://www.ncdc.noaa.gov/sst is not resolving as a resource (404).',
-                    'http://www.ncdc.noaa.gov/sst/ is not resolving as a resource (404).']
+        expected = [ 'http://doi.org/10.7289/V5SQ8XB5 is not resolving as a resource (404).',
+                     'http://www.ncdc.noaa.gov/sst is not resolving as a resource (404).',
+                     'http://www.ncdc.noaa.gov/sst/ is not resolving as a resource (404).']
         expected.sort()
         self.assertTrue(not validation.is_valid() and exns == expected,
                         msg='{} \n!= \n{}'.format(exns, expected))

lib/bald/tests/integration/test_cdl_rdfgraph.py

@@ -19,12 +19,24 @@ def test_array_reference(self):
             subprocess.check_call(['ncgen', '-o', tfile, cdl_file])
             root_container = bald.load_netcdf(tfile)
             ttl = root_container.rdfgraph().serialize(format='n3').decode("utf-8")
-            # with open(os.path.join(self.ttl_path, 'array_reference.ttl'), 'w') as sf:
+            # with open(os.path.join(self.ttl_path, 'array_reference2.ttl'), 'w') as sf:
             #     sf.write(ttl)
             with open(os.path.join(self.ttl_path, 'array_reference.ttl'), 'r') as sf:
                 expected_ttl = sf.read()
             self.assertEqual(expected_ttl, ttl)
 
+    def test_array_reference_with_baseuri(self):
+        with self.temp_filename('.nc') as tfile:
+            cdl_file = os.path.join(self.cdl_path, 'array_reference.cdl')
+            subprocess.check_call(['ncgen', '-o', tfile, cdl_file])
+            root_container = bald.load_netcdf(tfile, baseuri='http://example.org/base')
+            ttl = root_container.rdfgraph().serialize(format='n3').decode("utf-8")
+            #with open(os.path.join(self.ttl_path, 'array_reference_withbase.ttl'), 'w') as sf:
+            #     sf.write(ttl)
+            with open(os.path.join(self.ttl_path, 'array_reference_withbase.ttl'), 'r') as sf:
+                expected_ttl = sf.read()
+            self.assertEqual(expected_ttl, ttl)
+
     def test_multi_array_reference(self):
         with self.temp_filename('.nc') as tfile:
             cdl_file = os.path.join(self.cdl_path, 'multi_array_reference.cdl')
@@ -36,3 +48,15 @@ def test_multi_array_reference(self):
             with open(os.path.join(self.ttl_path, 'multi_array_reference.ttl'), 'r') as sf:
                 expected_ttl = sf.read()
             self.assertEqual(expected_ttl, ttl)
+
+    def test_ereefs(self):
+        with self.temp_filename('.nc') as tfile:
+            cdl_file = os.path.join(self.cdl_path, 'ereefs_gbr4_ncld.cdl')
+            subprocess.check_call(['ncgen', '-o', tfile, cdl_file])
+            root_container = bald.load_netcdf(tfile)
+            try:
+               g  = root_container.rdfgraph()
+               ttl = g.serialize(format='n3').decode("utf-8")
+            except TypeError:
+               self.fail("Test case could not convert ereefs CDL to RDF")
+