Merge pull request #85 from marqh/opt-in-ref

Opt in ref

Author: jyucsiro

README.md

@@ -2,7 +2,41 @@
 
 [![Build Status](https://api.travis-ci.org/repositories/binary-array-ld/bald.svg?branch=master)](http://travis-ci.org/binary-array-ld/bald/branches)
 
-A Python library for validating and managing binary array linked data files.
+A Python library for managing binary array linked data files.
+
+
+## Parsing
+
+This library processes payloads of various encodings and produces metadata graphs, in RDF, of the contents.  
+
+Input payloads are provided with an identity, elements within the payload are identified, with respect to that payload identity, unless they are otherwise identified, by prefix or by alias.
+
+Only Object values (attribute values) may be literals, consistent with the RDF interpretation. All Subjects and Predicates are identified with URIs. 
+
+Some format specific implementation exists, due to the different capabilities available from different encodings.
+
+### netCDF
+
+For netCDF files, no variable to variable reference definitions are provided by the API.
+
+For a variable to variable reference to be interpreted, an externally provided alias or predicate graph must explicitly state that a particular attribute has an rdfs:range of http://binary-array-ld.net/latest/Subject or a subclass of that class.
+
+netCDF parsing is working to support the emerging draft Open Geospaatial Consortium approach to netCDF-Linked-Data: https://github.com/opengeospatial/netCDF-Classic-LD/
+
+### HDF
+
+HDF files and APIs have an object reference, indicating a reference from one element in the file to another.
+
+These object refernces shall always be interpreted as references, regardless of the semantics of predicates (attribute names) provided by external graphs.
+
+
+## Validation
+
+This library provides some limited validation capabilities.
+
+It is expected that a valid input payload will be able to be processed into a graph.  
+
+Validation rules are limited in this implementation to require that provided HTTP URIs resolve.
 
 ## Pre-requisites
 

lib/bald/__init__.py

@@ -918,7 +918,22 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None):
         refs = reference_graph.query(qstr)
 
         non_ref_prefs = [str(ref[0]) for ref in list(refs)]
+
+        qstr = ('prefix bald: <http://binary-array-ld.net/latest/> '
+                'prefix skos: <http://www.w3.org/2004/02/skos/core#> '
+                'prefix owl: <http://www.w3.org/2002/07/owl#> '
+                'select ?s '
+                'where { '
+                '   {?s rdfs:range bald:Subject .} '
+                '  UNION '
+                '  {?s rdfs:range ?as . '
+                '  ?as rdfs:subClassOf bald:Subject .} '
+                '}')
 
+        refs = reference_graph.query(qstr)
+
+        ref_prefs = [str(ref[0]) for ref in list(refs)]
+
         # cycle again and find references
         for name in fhandle.variables:
             if name ==  prefix_var_name:
@@ -933,7 +948,7 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None):
 
             # for sattr in sattrs:
             for sattr in (sattr for sattr in sattrs if
-                          root_container.unpack_predicate(sattr) not in non_ref_prefs):
+                          root_container.unpack_predicate(sattr) in ref_prefs):
 
                 if (isinstance(sattrs[sattr], six.string_types) and
                     file_variables.get(sattrs[sattr])):

lib/bald/tests/integration/TTL/ProcessChain0300.ttl

@@ -18,7 +18,7 @@
     bald:isPrefixedBy "prefix_list" .
 
 this:gfsmos_process_chain a bald:Subject ;
-    this:OM_Process ( this:step1 this:step2 ) .
+    this:OM_Process "(step1 step2)" .
 
 this:step1 a bald:Subject ;
     this:LE_ProcessStep <https://codes.nws.noaa.gov/NumericalWeatherPrediction/Models/GFS13> ;

lib/bald/tests/integration/TTL/ereefs_gbr4_ncld.ttl

@@ -1,19 +1,20 @@
+@prefix CFTerms: <http://def.scitools.org.uk/CFTerms/> .
+@prefix NetCDF: <http://def.scitools.org.uk/NetCDF/> .
 @prefix bald: <http://binary-array-ld.net/latest/> .
+@prefix cf_sname: <http://vocab.nerc.ac.uk/standard_name/> .
 @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
 @prefix this: <file://CDL/ereefs_gbr4_ncld.cdl/> .
 @prefix xml: <http://www.w3.org/XML/1998/namespace> .
 @prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
 
 <file://CDL/ereefs_gbr4_ncld.cdl> a bald:Container ;
-    this:Conventions "CF-1.0" ;
     this:Run_ID 1.85 ;
     this:hasVocab "1" ;
     this:metadata_link <http://marlin.csiro.au/geonetwork/srv/eng/search?&uuid=72020224-f086-434a-bbe9-a222c8e5cf0d> ;
     this:paramfile "in.prm" ;
     this:paramhead "GBR 4km resolution grid" ;
     this:shoc_version "v1.1 rev(5249)" ;
-    this:title "GBR4 Hydro" ;
     bald:contains this:botz,
         this:eta,
         this:eta_time_ref,
@@ -33,89 +34,129 @@
         this:wspeed_v,
         this:wspeed_v_time_ref,
         this:zc ;
-    bald:isPrefixedBy "prefix_list" .
+    bald:isPrefixedBy "prefix_list" ;
+    NetCDF:Conventions "CF-1.0" ;
+    NetCDF:title "GBR4 Hydro" .
 
 this:botz a bald:Array ;
-    this:coordinates this:latitude,
-        this:longitude ;
-    this:long_name "Depth of sea-bed" ;
-    this:missing_value -99.0 ;
     this:outside "9999" ;
-    this:positive "down" ;
-    this:standard_name "depth" ;
-    this:units "metre" ;
-    bald:shape "(180, 600)" .
+    bald:shape "(180, 600)" ;
+    CFTerms:coordinates this:latitude,
+        this:longitude ;
+    CFTerms:missing_value -99.0 ;
+    CFTerms:positive "down" ;
+    CFTerms:standard_name <http://vocab.nerc.ac.uk/standard_name/depth/> ;
+    NetCDF:long_name "Depth of sea-bed" ;
+    NetCDF:units "metre" .
 
 this:eta a bald:Array ;
-    this:coordinates this:latitude,
-        this:longitude,
-        this:time ;
-    this:long_name "Surface elevation" ;
     this:medium_id <http://environment.data.gov.au/def/feature/ocean> ;
-    this:missing_value "1e+35" ;
-    this:positive "up" ;
     this:scaledQuantityKind_id <http://environment.data.gov.au/def/property/sea_surface_elevation> ;
-    this:standard_name "sea_surface_height_above_sea_level" ;
     this:substanceOrTaxon_id <http://environment.data.gov.au/def/feature/ocean_near_surface> ;
     this:unit_id <http://qudt.org/vocab/unit#Meter> ;
-    this:units "metre" ;
-    this:valid_range ( -10.0 10.0 ) ;
     bald:references this:eta_time_ref ;
-    bald:shape "(5, 180, 600)" .
+    bald:shape "(5, 180, 600)" ;
+    CFTerms:coordinates this:latitude,
+        this:longitude,
+        this:time ;
+    CFTerms:missing_value "1e+35" ;
+    CFTerms:positive "up" ;
+    CFTerms:standard_name <http://vocab.nerc.ac.uk/standard_name/sea_surface_height_above_sea_level/> ;
+    NetCDF:long_name "Surface elevation" ;
+    NetCDF:units "metre" ;
+    NetCDF:valid_range ( -10.0 10.0 ) .
 
 this:salt a bald:Array ;
-    this:coordinates this:latitude,
-        this:longitude,
-        this:time,
-        this:zc ;
-    this:long_name "Salinity" ;
     this:medium_id <http://environment.data.gov.au/def/feature/ocean> ;
-    this:missing_value "1e+35" ;
     this:scaledQuantityKind_id <http://environment.data.gov.au/def/property/practical_salinity> ;
     this:substanceOrTaxon_id <http://sweet.jpl.nasa.gov/2.2/matrWater.owl#SaltWater> ;
     this:unit_id <http://environment.data.gov.au/water/quality/def/unit/PSU> ;
-    this:units "PSU" ;
-    this:valid_range ( 0.0 40.0 ) ;
     bald:references this:salt_time_ref ;
-    bald:shape "(5, 47, 180, 600)" .
-
-this:temp a bald:Array ;
-    this:coordinates this:latitude,
+    bald:shape "(5, 47, 180, 600)" ;
+    CFTerms:coordinates this:latitude,
         this:longitude,
         this:time,
         this:zc ;
-    this:long_name "Temperature" ;
+    CFTerms:missing_value "1e+35" ;
+    NetCDF:long_name "Salinity" ;
+    NetCDF:units "PSU" ;
+    NetCDF:valid_range ( 0.0 40.0 ) .
+
+this:temp a bald:Array ;
     this:medium_id <http://environment.data.gov.au/def/feature/ocean> ;
-    this:missing_value "1e+35" ;
     this:scaledQuantityKind_id <http://environment.data.gov.au/def/property/sea_water_temperature> ;
     this:substanceOrTaxon_id <http://sweet.jpl.nasa.gov/2.2/matrWater.owl#SaltWater> ;
     this:unit_id <http://qudt.org/vocab/unit#DegreeCelsius> ;
-    this:units "degrees C" ;
-    this:valid_range ( 0.0 40.0 ) ;
     bald:references this:temp_time_ref ;
-    bald:shape "(5, 47, 180, 600)" .
+    bald:shape "(5, 47, 180, 600)" ;
+    CFTerms:coordinates this:latitude,
+        this:longitude,
+        this:time,
+        this:zc ;
+    CFTerms:missing_value "1e+35" ;
+    NetCDF:long_name "Temperature" ;
+    NetCDF:units "degrees C" ;
+    NetCDF:valid_range ( 0.0 40.0 ) .
+
+this:u a bald:Array ;
+    this:medium_id <http://environment.data.gov.au/def/feature/ocean> ;
+    this:scaledQuantityKind_id <http://environment.data.gov.au/def/property/sea_water_velocity_eastward> ;
+    this:substanceOrTaxon_id <http://environment.data.gov.au/def/feature/ocean_current> ;
+    this:unit_id <http://qudt.org/vocab/unit#MeterPerSecond> ;
+    this:vector_components "u v" ;
+    this:vector_name "Currents" ;
+    bald:references this:u_time_ref ;
+    bald:shape "(5, 47, 180, 600)" ;
+    CFTerms:coordinates this:latitude,
+        this:longitude,
+        this:time,
+        this:zc ;
+    CFTerms:missing_value "1e+35" ;
+    CFTerms:standard_name <http://vocab.nerc.ac.uk/standard_name/eastward_sea_water_velocity/> ;
+    NetCDF:long_name "Eastward current" ;
+    NetCDF:units "ms-1" ;
+    NetCDF:valid_range ( -100.0 100.0 ) .
+
+this:v a bald:Array ;
+    this:medium_id <http://environment.data.gov.au/def/feature/ocean> ;
+    this:scaledQuantityKind_id <http://environment.data.gov.au/def/property/sea_water_velocity_northward> ;
+    this:substanceOrTaxon_id <http://environment.data.gov.au/def/feature/ocean_current> ;
+    this:unit_id <http://qudt.org/vocab/unit#MeterPerSecond> ;
+    this:vector_components "u v" ;
+    this:vector_name "Currents" ;
+    bald:references this:v_time_ref ;
+    bald:shape "(5, 47, 180, 600)" ;
+    CFTerms:coordinates this:latitude,
+        this:longitude,
+        this:time,
+        this:zc ;
+    CFTerms:missing_value "1e+35" ;
+    CFTerms:standard_name <http://vocab.nerc.ac.uk/standard_name/northward_sea_water_velocity/> ;
+    NetCDF:long_name "Northward current" ;
+    NetCDF:units "ms-1" ;
+    NetCDF:valid_range ( -100.0 100.0 ) .
 
 this:wspeed_u a bald:Array ;
-    this:coordinates this:latitude,
+    bald:references this:wspeed_u_time_ref ;
+    bald:shape "(5, 180, 600)" ;
+    CFTerms:coordinates this:latitude,
         this:longitude,
         this:time ;
-    this:long_name "eastward_wind" ;
-    this:missing_value "1e+35" ;
-    this:units "ms-1" ;
-    this:valid_range ( -1000.0 1000.0 ) ;
-    bald:references this:wspeed_u_time_ref ;
-    bald:shape "(5, 180, 600)" .
+    CFTerms:missing_value "1e+35" ;
+    NetCDF:long_name "eastward_wind" ;
+    NetCDF:units "ms-1" ;
+    NetCDF:valid_range ( -1000.0 1000.0 ) .
 
 this:wspeed_v a bald:Array ;
-    this:coordinates this:latitude,
+    bald:references this:wspeed_v_time_ref ;
+    bald:shape "(5, 180, 600)" ;
+    CFTerms:coordinates this:latitude,
         this:longitude,
         this:time ;
-    this:long_name "northward_wind" ;
-    this:missing_value "1e+35" ;
-    this:units "ms-1" ;
-    this:valid_range ( -1000.0 1000.0 ) ;
-    bald:references this:wspeed_v_time_ref ;
-    bald:shape "(5, 180, 600)" .
+    CFTerms:missing_value "1e+35" ;
+    NetCDF:long_name "northward_wind" ;
+    NetCDF:units "ms-1" ;
+    NetCDF:valid_range ( -1000.0 1000.0 ) .
 
 this:eta_time_ref a bald:Reference,
         bald:Subject ;
@@ -132,25 +173,10 @@ this:temp_time_ref a bald:Reference,
     bald:array this:time ;
     bald:childBroadcast "(5, 1, 1, 1)" .
 
-this:u a bald:Array ;
-    this:coordinates this:latitude,
-        this:longitude,
-        this:time,
-        this:zc ;
-    this:long_name "Eastward current" ;
-    this:medium_id <http://environment.data.gov.au/def/feature/ocean> ;
-    this:missing_value "1e+35" ;
-    this:scaledQuantityKind_id <http://environment.data.gov.au/def/property/sea_water_velocity_eastward> ;
-    this:standard_name "eastward_sea_water_velocity" ;
-    this:substanceOrTaxon_id <http://environment.data.gov.au/def/feature/ocean_current> ;
-    this:unit_id <http://qudt.org/vocab/unit#MeterPerSecond> ;
-    this:units "ms-1" ;
-    this:valid_range ( -100.0 100.0 ) ;
-    this:vector_components this:u,
-        this:v ;
-    this:vector_name "Currents" ;
-    bald:references this:u_time_ref ;
-    bald:shape "(5, 47, 180, 600)" .
+this:u_time_ref a bald:Reference,
+        bald:Subject ;
+    bald:array this:time ;
+    bald:childBroadcast "(5, 1, 1, 1)" .
 
 this:v_time_ref a bald:Reference,
         bald:Subject ;
@@ -167,61 +193,36 @@ this:wspeed_v_time_ref a bald:Reference,
     bald:array this:time ;
     bald:childBroadcast "(5, 1, 1)" .
 
-this:u_time_ref a bald:Reference,
-        bald:Subject ;
-    bald:array this:time ;
-    bald:childBroadcast "(5, 1, 1, 1)" .
-
-this:v a bald:Array ;
-    this:coordinates this:latitude,
-        this:longitude,
-        this:time,
-        this:zc ;
-    this:long_name "Northward current" ;
-    this:medium_id <http://environment.data.gov.au/def/feature/ocean> ;
-    this:missing_value "1e+35" ;
-    this:scaledQuantityKind_id <http://environment.data.gov.au/def/property/sea_water_velocity_northward> ;
-    this:standard_name "northward_sea_water_velocity" ;
-    this:substanceOrTaxon_id <http://environment.data.gov.au/def/feature/ocean_current> ;
-    this:unit_id <http://qudt.org/vocab/unit#MeterPerSecond> ;
-    this:units "ms-1" ;
-    this:valid_range ( -100.0 100.0 ) ;
-    this:vector_components this:u_time_ref,
-        this:v ;
-    this:vector_name "Currents" ;
-    bald:references this:v_time_ref ;
-    bald:shape "(5, 47, 180, 600)" .
-
 this:zc a bald:Array ;
-    this:axis "Z" ;
     this:coordinate_type "Z" ;
-    this:long_name "Z coordinate" ;
-    this:positive "up" ;
-    this:units "m" ;
-    bald:shape "(47,)" .
+    bald:shape "(47,)" ;
+    CFTerms:axis "Z" ;
+    CFTerms:positive "up" ;
+    NetCDF:long_name "Z coordinate" ;
+    NetCDF:units "m" .
 
 this:latitude a bald:Array ;
-    this:coordinate_type this:latitude ;
-    this:long_name "Latitude" ;
+    this:coordinate_type "latitude" ;
     this:projection "geographic" ;
-    this:standard_name this:latitude ;
-    this:units "degrees_north" ;
-    bald:shape "(180, 600)" .
+    bald:shape "(180, 600)" ;
+    CFTerms:standard_name <http://vocab.nerc.ac.uk/standard_name/latitude/> ;
+    NetCDF:long_name "Latitude" ;
+    NetCDF:units "degrees_north" .
 
 this:longitude a bald:Array ;
-    this:coordinate_type this:longitude ;
-    this:long_name "Longitude" ;
+    this:coordinate_type "longitude" ;
     this:projection "geographic" ;
-    this:standard_name this:longitude ;
-    this:units "degrees_east" ;
-    bald:shape "(180, 600)" .
+    bald:shape "(180, 600)" ;
+    CFTerms:standard_name <http://vocab.nerc.ac.uk/standard_name/longitude/> ;
+    NetCDF:long_name "Longitude" ;
+    NetCDF:units "degrees_east" .
 
 this:time a bald:Array,
         bald:Reference ;
-    this:coordinate_type this:time ;
-    this:long_name "Time" ;
-    this:standard_name this:time ;
-    this:units "days since 1990-01-01 00:00:00 +10" ;
+    this:coordinate_type "time" ;
     bald:array this:time ;
-    bald:shape "(5,)" .
+    bald:shape "(5,)" ;
+    CFTerms:standard_name <http://vocab.nerc.ac.uk/standard_name/time/> ;
+    NetCDF:long_name "Time" ;
+    NetCDF:units "days since 1990-01-01 00:00:00 +10" .