Refactoring alias and prefix handling to variable

Author: jyucsiro

lib/bald/__init__.py

@@ -240,6 +240,7 @@ def __getitem__(self, item):
 
     def check_uri(self, uri):
         result = False
+        #print("Checking uri: " + uri)
         if self[uri].status_code == 200:
             result = True
         return result
@@ -438,6 +439,11 @@ def rdfgraph(self):
         """
         graph = rdflib.Graph()
         graph.bind('bald', 'http://binary-array-ld.net/latest/')
+        for prefix_name in self._prefixes:
+           #strip the double underscore suffix
+           new_name = prefix_name[:-2]
+
+           graph.bind(new_name, self._prefixes[prefix_name])
         graph = self.rdfnode(graph)
 
         return graph
@@ -537,22 +543,34 @@ def load_netcdf(afilepath, uri=None):
     """
 
     with load(afilepath) as fhandle:
-        prefix_group = (fhandle[fhandle.bald__isPrefixedBy] if
+        prefix_var_name  = None
+        if hasattr(fhandle, 'bald__isPrefixedBy'):
+           prefix_var_name  = fhandle.bald__isPrefixedBy
+
+        prefix_var = (fhandle[fhandle.bald__isPrefixedBy] if
                         hasattr(fhandle, 'bald__isPrefixedBy') else {})
         prefixes = {}
-        if prefix_group:
-            prefixes = (dict([(prefix, getattr(prefix_group, prefix)) for
-                              prefix in prefix_group.ncattrs()]))
+        if prefix_var is not {} :
+            prefixes = (dict([(prefix, getattr(prefix_var, prefix)) for
+                              prefix in prefix_var.ncattrs()]))
         else:
             for k in fhandle.ncattrs():
                 if k.endswith('__'):
                     prefixes[k] = getattr(fhandle, k)
-        alias_group = (fhandle[fhandle.bald__isAliasedBy]
+
+        #print(prefixes)
+
+        alias_var_name = None
+        if hasattr(fhandle, 'bald__isAliasedBy'):
+           alias_var_name  = fhandle.bald__isAliasedBy
+
+        alias_var = (fhandle[fhandle.bald__isAliasedBy]
                        if hasattr(fhandle, 'bald__isAliasedBy') else {})
         aliases = {}
-        if alias_group:
-            aliases = (dict([(alias, getattr(alias_group, alias))
-                             for alias in alias_group.ncattrs()]))
+        if alias_var != {}:
+            aliases = (dict([(alias, getattr(alias_var, alias))
+                             for alias in alias_var.ncattrs()]))
+        #print(aliases)
 
         attrs = {}
         for k in fhandle.ncattrs():
@@ -567,6 +585,10 @@ def load_netcdf(afilepath, uri=None):
         root_container.attrs['bald__contains'] = []
         file_variables = {}
         for name in fhandle.variables:
+            #print(name)
+            if name ==  prefix_var_name or name == alias_var_name:
+                #print("Skipping " + name)
+                continue
 
             sattrs = fhandle.variables[name].__dict__.copy()
             # inconsistent use of '/'; fix it
@@ -590,6 +612,10 @@ def load_netcdf(afilepath, uri=None):
 
         # cycle again and find references
         for name in fhandle.variables:
+            if name ==  prefix_var_name or name == alias_var_name:
+                #print("Skipping " + name)
+                continue
+
             var = file_variables[name]
             # reverse lookup based on type to be added
             lookups = ['bald__references', 'bald__array']

lib/bald/tests/integration/CDL/array_alias_v2.cdl

@@ -0,0 +1,58 @@
+netcdf tmpMwXy8U {
+dimensions:
+	pdim0 = 11 ;
+	pdim1 = 17 ;
+variables:
+	int prefix_list ;
+  		prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
+  		prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
+
+	int alias_list ;
+        	alias_list:SDN_ParameterDiscoveryCode = "http://vocab.nerc.ac.uk/isoCodelists/sdnCodelists/cdicsrCodeList.xml#SDN_ParameterDiscoveryCode" ;
+	        alias_list:BactTaxaAbundSed = "http://vocab.nerc.ac.uk/collection/P02/current/BAUC/" ;
+		alias_list:standard_name = "https://def.scitools.org.uk/CFTerms/standard_name" ;
+		alias_list:air_temperature = "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/" ;
+
+	int parent_variable(pdim0, pdim1) ;
+		parent_variable:rdf__type = "bald__Array" ;
+		parent_variable:SDN_ParameterDiscoveryCode = "BactTaxaAbundSed" ;
+		parent_variable:submursible_name = "Nautilus" ;
+
+	int temp(pdim0, pdim1) ;
+	        temp:standard_name = "air_temperature" ;
+
+// global attributes:
+		:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
+		:rdf__type = "bald__Container" ;
+		:bald__isPrefixedBy = "prefix_list" ;
+		:bald__isAliasedBy = "alias_list" ;
+data:
+
+ parent_variable =
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;
+
+ temp =
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;
+
+
+}

lib/bald/tests/integration/CDL/array_prefix_v2.cdl

@@ -0,0 +1,55 @@
+netcdf tmpMwXy8U {
+dimensions:
+	pdim0 = 11 ;
+	pdim1 = 17 ;
+variables:
+	int prefix_list ;
+  		prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
+  		prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
+        	prefix_list:sdn__ = "http://vocab.nerc.ac.uk/isoCodelists/sdnCodelists/cdicsrCodeList.xml#" ;
+	        prefix_list:sdn-vocab__= "http://vocab.nerc.ac.uk/collection/P02/current/" ;
+		prefix_list:cf__ = "http://def.scitools.org.uk/CFTerms/" ;
+		prefix_list:cfsn__ = "http://vocab.nerc.ac.uk/collection/P07/current/CFSN0023/" ;
+
+	int parent_variable(pdim0, pdim1) ;
+		parent_variable:rdf__type = "bald__Array" ;
+		parent_variable:sdn__SDN_ParameterDiscoveryCode = "sdn-vocab__BAUC" ;
+		parent_variable:submursible_name = "Nautilus" ;
+
+	int temp(pdim0, pdim1) ;
+	        temp:standard_name = "air_temperature" ;
+
+// global attributes:
+		:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
+		:rdf__type = "bald__Container" ;
+		:bald__isPrefixedBy = "prefix_list" ;
+data:
+
+ parent_variable =
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;
+
+ temp =
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;
+
+
+}

lib/bald/tests/integration/CDL/array_prefix_v2_full.cdl

@@ -0,0 +1,57 @@
+netcdf tmpMwXy8U {
+dimensions:
+	pdim0 = 11 ;
+	pdim1 = 17 ;
+variables:
+	int prefix_list ;
+  		prefix_list:bald__ = "http://binary-array-ld.net/latest/" ;
+  		prefix_list:rdf__ = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ;
+        	prefix_list:sdn__ = "http://vocab.nerc.ac.uk/isoCodelists/sdnCodelists/cdicsrCodeList.xml#" ;
+	        prefix_list:sdn-vocab__= "http://vocab.nerc.ac.uk/collection/P02/current/" ;
+		prefix_list:cf__ = "http://def.scitools.org.uk/CFTerms/" ;
+		prefix_list:cfsn-mmi__ = "http://mmisw.org/ont/cf/parameter/" ;
+		prefix_list:cfsn-nerc__ = "http://vocab.nerc.ac.uk/collection/P07/current/";
+
+	int parent_variable(pdim0, pdim1) ;
+		parent_variable:rdf__type = "bald__Array" ;
+		parent_variable:sdn__SDN_ParameterDiscoveryCode = "sdn-vocab__BAUC" ;
+		parent_variable:submursible_name = "Nautilus" ;
+
+	int temp(pdim0, pdim1) ;
+	        temp:cf__standard_name = "cfsn-mmi__air_temperature" ;
+		//temp:cf__standard_name = "cfsn-nerc__CFSN0023";
+
+// global attributes:
+		:_NCProperties = "version=1|netcdflibversion=4.4.1|hdf5libversion=1.8.17" ;
+		:rdf__type = "bald__Container" ;
+		:bald__isPrefixedBy = "prefix_list" ;
+data:
+
+ parent_variable =
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;
+
+ temp =
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+  _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ ;
+
+
+}

lib/bald/tests/integration/test_cdl_v2.py

@@ -0,0 +1,40 @@
+import glob
+import os
+import subprocess
+import unittest
+
+import netCDF4
+import numpy as np
+
+import bald
+from bald.tests import BaldTestCase
+
+
+class Test(BaldTestCase):
+    def setUp(self):
+        self.cdl_path = os.path.join(os.path.dirname(__file__), 'CDL')
+
+
+def test_prefix_v2(self):
+    """Test prefix version 2 style """
+    with self.temp_filename('.nc') as tfile:
+        cdl_file = os.path.join(self.cdl_path, 'array_prefix_v2.cdl')
+        print(cdl_file)
+        subprocess.check_call(['ncgen', '-o', tfile, cdl_file])
+        validation = bald.validate_netcdf(tfile)
+        exns = validation.exceptions()
+        self.assertTrue(validation.is_valid(), msg='{} != []'.format(exns))
+
+setattr(Test, 'test_prefix_v2', test_prefix_v2)
+
+def test_prefix_v2_full(self):
+    """Test prefix version 2 style - full example"""
+    with self.temp_filename('.nc') as tfile:
+        cdl_file = os.path.join(self.cdl_path, 'array_prefix_v2_full.cdl')
+        print(cdl_file)
+        subprocess.check_call(['ncgen', '-o', tfile, cdl_file])
+        validation = bald.validate_netcdf(tfile)
+        exns = validation.exceptions()
+        self.assertTrue(validation.is_valid(), msg='{} != []'.format(exns))
+
+setattr(Test, 'test_prefix_v2_full', test_prefix_v2_full)

lib/bald/validation.py

@@ -90,9 +90,11 @@ def _check_uri(uri, exceptions):
                 exceptions.append(msg)
             return exceptions
 
+        ''' Skip checking prefixes as whole graphs could be big!
         for pref, uri in self.subject.prefixes().items():
             exceptions = _check_uri(self.subject.unpack_uri(uri),
                                     exceptions)
+        '''
         for alias, uri in self.subject.aliases.items():
             exceptions = _check_uri(self.subject.unpack_uri(uri),
                                     exceptions)