Skip to content

Commit 523c756

Browse files
marqhmarqh
authored
Merge pull request #2 from marqh/coordss
Coord start and end
2 parents a4bfb2c + 455ee18 commit 523c756

File tree

9 files changed

+335
-45
lines changed

9 files changed

+335
-45
lines changed

‎.travis.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@ install:
1717
- conda install --quiet --file requirements.txt
1818
- conda list
1919
- conda info -a
20+
- wget https://github.com/marqh/terra/archive/master.zip
21+
- unzip master.zip
22+
- cd terra-master
23+
- python setup.py --quiet install
24+
- cd ..
2025
- python setup.py --quiet install
2126

2227
script:

‎lib/bald/__init__.py

Lines changed: 80 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import contextlib
22
import copy
33
import re
4+
import time
45

56
import h5py
67
import jinja2
@@ -11,6 +12,12 @@
1112
import requests
1213
import six
1314

15+
try:
16+
import terra.datetime
17+
terra_imp = True
18+
except ImportError:
19+
terra_imp = False
20+
1421
import bald.validation as bv
1522

1623
__version__ = '0.3'
@@ -235,19 +242,18 @@ def __getitem__(self, item):
235242
if not self.is_http_uri(item):
236243
raise ValueError('{} is not a HTTP URI.'.format(item))
237244
if item not in self.cache:
238-
# import datetime
239-
# now = datetime.datetime.utcnow()
240-
# print('\ndownloading: {}'.format(item))
241-
self.cache[item] = requests.get(item)
245+
# now = time.time()
242246
try:
243-
# Attempt content negotiation, but pass if problems occur.
247+
# print('trying: {}'.format(item))
248+
244249
headers = {'Accept': 'application/rdf+xml'}
245-
self.cache[item] = requests.get(item, headers=headers)
250+
self.cache[item] = requests.get(item, headers=headers, timeout=7)
246251
except Exception:
247-
pass
248-
# then = datetime.datetime.utcnow()
249-
# print('{}s'.format((then-now).total_seconds()))
252+
# print('retrying: {}'.format(item))
253+
headers = {'Accept': 'text/html'}
254+
self.cache[item] = requests.get(item, headers=headers, timeout=7)
250255

256+
# print('in {} seconds'.format(time.time() - then))
251257
return self.cache[item]
252258

253259
def check_uri(self, uri):
@@ -446,7 +452,8 @@ def _graph_elem_attrs(self, remaining_attrs):
446452
else:
447453
kstr = '{key}: '.format(key=attr)
448454
vals = remaining_attrs[attr]
449-
if isinstance(vals, six.string_types):
455+
if (isinstance(vals, six.string_types) or
456+
isinstance(vals, np.ma.core.MaskedConstant)):
450457
vuri = self.unpack_rdfobject(vals, predicate=attr_uri)
451458
if is_http_uri(vuri):
452459
vstr = self.link_template
@@ -747,9 +754,55 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None):
747754

748755
# netCDF coordinate variable special case
749756
if (len(fhandle.variables[name].dimensions) == 1 and
750-
fhandle.variables[name].dimensions[0] == name):
757+
fhandle.variables[name].dimensions[0] == name and
758+
len(fhandle.variables[name]) > 0):
751759
sattrs['bald__array'] = name
752760
sattrs['rdf__type'] = 'bald__Reference'
761+
762+
sattrs['bald__first_value'] = fhandle.variables[name][0]
763+
if len(fhandle.variables[name]) > 1:
764+
sattrs['bald__last_value'] = fhandle.variables[name][-1]
765+
766+
# datetime special case
767+
if 'units' in fhandle.variables[name].ncattrs() and terra_imp:
768+
ustr = fhandle.variables[name].getncattr('units')
769+
pattern = '^([a-z]+) since ([0-9T:\\. -]+)'
770+
771+
amatch = re.match(pattern, ustr)
772+
if amatch:
773+
quantity = amatch.group(1)
774+
origin = amatch.group(2)
775+
ig = terra.datetime.ISOGregorian()
776+
tog = terra.datetime.parse_datetime(origin,
777+
calendar=ig)
778+
dtype = '{}{}'.format(fhandle.variables[name].dtype.kind,
779+
fhandle.variables[name].dtype.itemsize)
780+
fv = netCDF4.default_fillvals.get(dtype)
781+
if fhandle.variables[name][0] == fv:
782+
first = np.ma.MaskedArray(fhandle.variables[name][0],
783+
mask=True)
784+
else:
785+
first = fhandle.variables[name][0]
786+
787+
edate_first = terra.datetime.EpochDateTimes(first,
788+
quantity,
789+
epoch=tog)
790+
791+
sattrs['bald__first_value'] = str(edate_first)
792+
if len(fhandle.variables[name]) > 1:
793+
if fhandle.variables[name][0] == fv:
794+
last = np.ma.MaskedArray(fhandle.variables[name][-1],
795+
mask=True)
796+
else:
797+
last = fhandle.variables[name][-1]
798+
edate_last = terra.datetime.EpochDateTimes(last,
799+
quantity,
800+
epoch=tog)
801+
sattrs['bald__last_value'] = str(edate_last)
802+
803+
804+
805+
753806

754807
if fhandle.variables[name].shape:
755808
sattrs['bald__shape'] = fhandle.variables[name].shape
@@ -768,15 +821,28 @@ def load_netcdf(afilepath, baseuri=None, alias_dict=None, cache=None):
768821
response = cache['http://binary-array-ld.net/latest']
769822
reference_graph.parse(data=response.text, format='xml')
770823

771-
# reference_graph.parse('http://binary-array-ld.net/latest?_format=ttl')
824+
# # reference_graph.parse('http://binary-array-ld.net/latest?_format=ttl')
825+
# qstr = ('prefix bald: <http://binary-array-ld.net/latest/> '
826+
# 'prefix skos: <http://www.w3.org/2004/02/skos/core#> '
827+
# 'select ?s '
828+
# 'where { '
829+
# ' ?s rdfs:range ?type . '
830+
# 'filter(?type != rdfs:Literal) '
831+
# 'filter(?type != skos:Concept) '
832+
# '}')
833+
834+
# refs_ = reference_graph.query(qstr)
835+
772836
qstr = ('prefix bald: <http://binary-array-ld.net/latest/> '
773837
'prefix skos: <http://www.w3.org/2004/02/skos/core#> '
838+
'prefix owl: <http://www.w3.org/2002/07/owl#> '
774839
'select ?s '
775840
'where { '
776841
' ?s rdfs:range ?type . '
777-
'filter(?type != rdfs:Literal) '
778-
'filter(?type != skos:Concept) '
842+
' ?type rdf:type ?rtype . '
843+
'filter(?rtype = owl:Class) '
779844
'}')
845+
780846
refs = reference_graph.query(qstr)
781847

782848
ref_prefs = [str(ref[0]) for ref in list(refs)]

‎lib/bald/tests/integration/CDL/GEMS_CO2_Apr2006.cdl

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
netcdf GEMS_CO2_Apr2006 {
2+
dimensions:
3+
longitude = 360 ;
4+
latitude = 181 ;
5+
levelist = 60 ;
6+
time = 1 ;
7+
variables:
8+
float longitude(longitude) ;
9+
longitude:units = "degrees_east" ;
10+
longitude:standard_name = "longitude" ;
11+
float latitude(latitude) ;
12+
latitude:units = "degrees_north" ;
13+
latitude:standard_name = "latitude" ;
14+
int levelist(levelist) ;
15+
levelist:long_name = "model_level_number" ;
16+
int time(time) ;
17+
time:units = "hours since 1900-01-01 00:00:0.0" ;
18+
time:standard_name = "time" ;
19+
short co2(time, levelist, latitude, longitude) ;
20+
co2:scale_factor = 0.000981685145029486 ;
21+
co2:add_offset = 403.192219379918 ;
22+
co2:_FillValue = -32767s ;
23+
co2:missing_value = -32767s ;
24+
co2:units = "kg kg**-1" ;
25+
co2:long_name = "Carbon Dioxide" ;
26+
co2:standard_name = "mass_fraction_of_carbon_dioxide_in_air" ;
27+
short lnsp(time, levelist, latitude, longitude) ;
28+
lnsp:scale_factor = 1.03952457840347e-05 ;
29+
lnsp:add_offset = 11.2087164280841 ;
30+
lnsp:_FillValue = -32767s ;
31+
lnsp:missing_value = -32767s ;
32+
lnsp:long_name = "Logarithm of surface pressure" ;
33+
34+
// global attributes:
35+
:Conventions = "CF-1.0" ;
36+
37+
data:
38+
39+
longitude = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
40+
18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
41+
36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
42+
54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
43+
72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
44+
90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105,
45+
106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
46+
120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133,
47+
134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147,
48+
148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161,
49+
162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175,
50+
176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189,
51+
190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203,
52+
204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217,
53+
218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231,
54+
232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245,
55+
246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259,
56+
260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273,
57+
274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287,
58+
288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301,
59+
302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315,
60+
316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329,
61+
330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343,
62+
344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357,
63+
358, 359 ;
64+
65+
latitude = 90, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 79, 78, 77, 76, 75,
66+
74, 73, 72, 71, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, 60, 59, 58, 57,
67+
56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39,
68+
38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21,
69+
20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
70+
-1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -16,
71+
-17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -29, -30,
72+
-31, -32, -33, -34, -35, -36, -37, -38, -39, -40, -41, -42, -43, -44,
73+
-45, -46, -47, -48, -49, -50, -51, -52, -53, -54, -55, -56, -57, -58,
74+
-59, -60, -61, -62, -63, -64, -65, -66, -67, -68, -69, -70, -71, -72,
75+
-73, -74, -75, -76, -77, -78, -79, -80, -81, -82, -83, -84, -85, -86,
76+
-87, -88, -89, -90 ;
77+
78+
levelist = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
79+
19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
80+
37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54,
81+
55, 56, 57, 58, 59, 60 ;
82+
83+
time = 931344 ;
84+
}

0 commit comments

Comments
 (0)