Add I/O for masked Traces

Make backwards compatible with older ASDF versions


I/O of masked arrays of float/integer data.

Ignore some test files.

Author: Malcolm White

.gitignore

@@ -1,3 +1,6 @@
 *.egg-info
 doc/_build/
 .cache
+*.pyc
+.coverage
+.pytest_cache

pyasdf/asdf_data_set.py

@@ -37,7 +37,6 @@
 import prov
 import prov.model
 
-
 # Minimum compatibility wrapper between Python 2 and 3.
 try:
     filter = itertools.ifilter
@@ -811,7 +810,13 @@ def _get_waveform(self, waveform_name, starttime=None, endtime=None):
         data = self.__file["Waveforms"]["%s.%s" % (network, station)][
             waveform_name]
 
-        tr = obspy.Trace(data=data[idx_start: idx_end])
+        if "mask" in data.attrs and data.attrs["mask"] is not False:
+            _data = np.ma.masked_values(data[idx_start: idx_end],
+                                        data.attrs["mask"])
+        else:
+            _data = data[idx_start: idx_end]
+
+        tr = obspy.Trace(data=_data)
         tr.stats.starttime = data_starttime
         tr.stats.sampling_rate = data.attrs["sampling_rate"]
         tr.stats.network = network
@@ -1167,6 +1172,8 @@ def add_waveforms(self, waveform, tag, event_id=None, origin_id=None,
 
         # Actually add the data.
         for trace in waveform:
+            if isinstance(trace.data, np.ma.masked_array):
+                self.__set_masked_array_fill_value(trace)
             # Complicated multi-step process but it enables one to use
             # parallel I/O with the same functions.
             info = self._add_trace_get_collective_information(
@@ -1179,6 +1186,16 @@ def add_waveforms(self, waveform, tag, event_id=None, origin_id=None,
             self._add_trace_write_collective_information(info)
             self._add_trace_write_independent_information(info, trace)
 
+    def __set_masked_array_fill_value(self, trace):
+        if trace.data.dtype.kind in ("i", "u"):
+            _info = np.iinfo
+        elif trace.data.dtype.kind == "f" :
+            _info = np.finfo
+        else:
+            raise(NotImplementedError("fill value for dtype %s not defined"\
+                    % trace.data.dtype))
+        trace.data.set_fill_value(_info(trace.data.dtype).min)
+
     def __parse_and_validate_tag(self, tag):
         tag = tag.strip()
         if tag.lower() == "stationxml":
@@ -1295,7 +1312,7 @@ def _add_trace_write_independent_information(self, info, trace):
         :param trace:
         :return:
         """
-        self._waveform_group[info["data_name"]][:] = trace.data
+        self._waveform_group[info["data_name"]][:] = np.ma.filled(trace.data)
 
     def _add_trace_write_collective_information(self, info):
         """
@@ -1367,6 +1384,12 @@ def _add_trace_get_collective_information(
         else:
             fletcher32 = True
 
+        # Determine appropriate mask value.
+        if not isinstance(trace.data, np.ma.masked_array):
+            _mask = np.bool(False)
+        else:
+            _mask = trace.data.fill_value
+
         info = {
             "station_name": station_name,
             "data_name": group_name,
@@ -1384,7 +1407,8 @@ def _add_trace_get_collective_information(
                 # Starttime is the epoch time in nanoseconds.
                 "starttime":
                     int(round(trace.stats.starttime.timestamp * 1.0E9)),
-                "sampling_rate": trace.stats.sampling_rate
+                "sampling_rate": trace.stats.sampling_rate,
+                "mask": _mask
             }
         }