Merge pull request #45 from jes-p/master

Add microseconds to waveform name if necessary.

Author: krischer

.travis.yml

@@ -2,7 +2,6 @@ language: python
 
 python:
   - "2.7"
-  - "3.4"
   - "3.5"
   - "3.6"
 

pyasdf/asdf_data_set.py

@@ -17,6 +17,7 @@
 
 import collections
 import copy
+from distutils.version import LooseVersion
 import io
 import itertools
 import math
@@ -198,7 +199,7 @@ def __init__(self, filename, compression="gzip-3", shuffle=True,
 
         # Deal with the file format version. `format_version` is either None
         # or valid (this is checked above).
-        __most_recent_version = "1.0.1"
+        __most_recent_version = "1.0.2"
         # Case 1: Already some kind of format version in the file.
         if "file_format_version" in self.__file.attrs:
             version_in_file = self.__file.attrs["file_format_version"].decode()
@@ -234,6 +235,10 @@ def __init__(self, filename, compression="gzip-3", shuffle=True,
         # Just a final safety check - should not be able to fail!
         assert self.asdf_format_version in SUPPORTED_FORMAT_VERSIONS
 
+        # Useful for programmatic checks.
+        self._loose_asdf_format_version = \
+            LooseVersion(self.asdf_format_version)
+
         # Create the waveform and provenance groups if mode is not "r".
         if "Waveforms" not in self.__file and mode != "r":
             self.__file.create_group("Waveforms")
@@ -1201,6 +1206,13 @@ def __parse_waveform_input_and_validate(self, waveform):
                                     "integers and 4 and 8 byte floating point "
                                     "numbers." %
                                     trace.data.dtype.name)
+                elif self.asdf_format_version == "1.0.2":
+                    raise TypeError("The trace's dtype ('%s') is not allowed "
+                                    "inside ASDF 1.0.2. Allowed are little "
+                                    "and big endian 2, 4, and 8 byte signed "
+                                    "integers and 4 and 8 byte floating point "
+                                    "numbers." %
+                                    trace.data.dtype.name)
                 else:  # pragma: no cover
                     raise NotImplementedError
         return waveform
@@ -1294,10 +1306,19 @@ def _add_trace_write_collective_information(self, info):
             ds.attrs[key] = value
 
     def __get_waveform_ds_name(self, net, sta, loc, cha, start, end, tag):
+        fmt_string = "%Y-%m-%dT%H:%M:%S"
+        s = start.strftime(fmt_string)
+        e = end.strftime(fmt_string)
+
+        # Add nanoseconds if two waveforms start in the same second. Only do
+        # so for recent ASDF versions.
+        if s == e and self._loose_asdf_format_version >= LooseVersion("1.0.2"):
+            s += "." + "%09i" % (start._ns % int(1e9))
+            e += "." + "%09i" % (end._ns % int(1e9))
+
         return "{net}.{sta}.{loc}.{cha}__{start}__{end}__{tag}".format(
             net=net, sta=sta, loc=loc, cha=cha,
-            start=start.strftime("%Y-%m-%dT%H:%M:%S"),
-            end=end.strftime("%Y-%m-%dT%H:%M:%S"),
+            start=s, end=e,
             tag=tag)
 
     def _add_trace_get_collective_information(

pyasdf/header.py

@@ -39,7 +39,7 @@
 
 
 FORMAT_NAME = "ASDF"
-SUPPORTED_FORMAT_VERSIONS = ("1.0.0", "1.0.1")
+SUPPORTED_FORMAT_VERSIONS = ("1.0.0", "1.0.1", "1.0.2")
 
 
 # Regular expression for allowed filenames within the provenance group.
@@ -55,6 +55,11 @@
               np.dtype("<f4"), np.dtype(">f4"),
               np.dtype("<f8"), np.dtype(">f8")),
     "1.0.1": (np.dtype("<i2"), np.dtype(">i2"),
+              np.dtype("<i4"), np.dtype(">i4"),
+              np.dtype("<i8"), np.dtype(">i8"),
+              np.dtype("<f4"), np.dtype(">f4"),
+              np.dtype("<f8"), np.dtype(">f8")),
+    "1.0.2": (np.dtype("<i2"), np.dtype(">i2"),
               np.dtype("<i4"), np.dtype(">i4"),
               np.dtype("<i8"), np.dtype(">i8"),
               np.dtype("<f4"), np.dtype(">f4"),

pyasdf/tests/test_asdf_data_set.py

@@ -264,7 +264,7 @@ def test_assert_format_and_version_number_are_written(tmpdir):
     # Open again and assert name and version number.
     with h5py.File(asdf_filename, "r") as hdf5_file:
         assert hdf5_file.attrs["file_format_version"].decode() \
-           == "1.0.1"
+           == "1.0.2"
         assert hdf5_file.attrs["file_format"].decode() == FORMAT_NAME
 
 
@@ -506,16 +506,25 @@ def test_format_version_handling(tmpdir):
 
     # Create.
     with ASDFDataSet(filename) as ds:
-        assert ds.asdf_format_version_in_file == "1.0.1"
-        assert ds.asdf_format_version == "1.0.1"
+        assert ds.asdf_format_version_in_file == "1.0.2"
+        assert ds.asdf_format_version == "1.0.2"
     # Open again.
     with ASDFDataSet(filename) as ds:
-        assert ds.asdf_format_version_in_file == "1.0.1"
-        assert ds.asdf_format_version == "1.0.1"
+        assert ds.asdf_format_version_in_file == "1.0.2"
+        assert ds.asdf_format_version == "1.0.2"
 
     os.remove(filename)
 
     # Directly specify it.
+    with ASDFDataSet(filename, format_version="1.0.2") as ds:
+        assert ds.asdf_format_version_in_file == "1.0.2"
+        assert ds.asdf_format_version == "1.0.2"
+    with ASDFDataSet(filename) as ds:
+        assert ds.asdf_format_version_in_file == "1.0.2"
+        assert ds.asdf_format_version == "1.0.2"
+
+    os.remove(filename)
+
     with ASDFDataSet(filename, format_version="1.0.1") as ds:
         assert ds.asdf_format_version_in_file == "1.0.1"
         assert ds.asdf_format_version == "1.0.1"
@@ -558,17 +567,17 @@ def test_format_version_handling(tmpdir):
         ds._ASDFDataSet__file.attrs["file_format_version"] = \
             ds._zeropad_ascii_string("x.x.x")
         assert ds.asdf_format_version_in_file == "x.x.x"
-        assert ds.asdf_format_version == "1.0.1"
+        assert ds.asdf_format_version == "1.0.2"
     with warnings.catch_warnings(record=True) as w:
         warnings.simplefilter("always")
         with ASDFDataSet(filename) as ds:
             assert ds.asdf_format_version_in_file == "x.x.x"
-            assert ds.asdf_format_version == "1.0.1"
+            assert ds.asdf_format_version == "1.0.2"
     assert w[0].message.args[0] == (
         "The file claims an ASDF version of x.x.x. This version of pyasdf "
-        "only supports versions: 1.0.0, 1.0.1. All following write operations "
-        "will use version 1.0.1 - other tools might not be able to read "
-        "the files again - proceed with caution.")
+        "only supports versions: 1.0.0, 1.0.1, 1.0.2. All following write "
+        "operations will use version 1.0.2 - other tools might not be able to "
+        "read the files again - proceed with caution.")
     # Again but force version.
     os.remove(filename)
     # Both can also differ.
@@ -583,17 +592,17 @@ def test_format_version_handling(tmpdir):
             assert ds.asdf_format_version == "1.0.0"
     assert w[0].message.args[0] == (
         "The file claims an ASDF version of x.x.x. This version of pyasdf "
-        "only supports versions: 1.0.0, 1.0.1. All following write operations "
-        "will use version 1.0.0 - other tools might not be able to read "
-        "the files again - proceed with caution.")
+        "only supports versions: 1.0.0, 1.0.1, 1.0.2. All following write "
+        "operations will use version 1.0.0 - other tools might not be able to "
+        "read the files again - proceed with caution.")
 
     # Unsupported version number.
     os.remove(filename)
     with pytest.raises(ASDFValueError) as err:
         ASDFDataSet(filename, format_version="x.x.x")
     assert err.value.args[0] == (
         "ASDF version 'x.x.x' is not supported. Supported versions: 1.0.0, "
-        "1.0.1")
+        "1.0.1, 1.0.2")
     # No file should be created.
     assert not os.path.exists(filename)
 
@@ -605,8 +614,8 @@ def test_format_version_handling(tmpdir):
     with warnings.catch_warnings(record=True) as w:
         warnings.simplefilter("always")
         with ASDFDataSet(filename) as ds:
-            assert ds.asdf_format_version_in_file == "1.0.1"
-            assert ds.asdf_format_version == "1.0.1"
+            assert ds.asdf_format_version_in_file == "1.0.2"
+            assert ds.asdf_format_version == "1.0.2"
     assert w[0].message.args[0] == (
         "No file format version given in file '%s'. The program will "
         "continue but the result is undefined." % os.path.abspath(filename))
@@ -3013,3 +3022,54 @@ def test_get_waveform_attributes(example_data_set):
                 'sampling_rate': 40.0,
                 'starttime': 1369374000000000000}
         }
+
+
+def test_datesets_with_less_then_1_second_length(tmpdir):
+    asdf_filename = os.path.join(tmpdir.strpath, "test.h5")
+
+    tr = obspy.Trace(np.linspace(0, 1, 777),
+                     header={"network": "AA", "station": "BB", "location": "",
+                             "channel": "000",
+                             "starttime": obspy.UTCDateTime(38978345.3445843)})
+
+    # Don't use nano-seconds if longer than one second.
+    with ASDFDataSet(asdf_filename) as ds:
+        tr.stats.sampling_rate = 1.0
+        ds.add_waveforms(tr, tag="test")
+        dataset_name = ds.waveforms["AA.BB"].list()[0]
+    os.remove(asdf_filename)
+
+    assert dataset_name == \
+        "AA.BB..000__1971-03-28T03:19:05__1971-03-28T03:32:01__test"
+
+    # Do, if shorter than one second.
+    with ASDFDataSet(asdf_filename) as ds:
+        tr.stats.sampling_rate = 474505737
+        ds.add_waveforms(tr, tag="test")
+        dataset_name = ds.waveforms["AA.BB"].list()[0]
+    os.remove(asdf_filename)
+
+    assert dataset_name == (
+        "AA.BB..000__1971-03-28T03:19:05.344584304__"
+        "1971-03-28T03:19:05.344585939__test")
+
+    # Don't do it for older versions to not write invalid files.
+    with ASDFDataSet(asdf_filename, format_version="1.0.1") as ds:
+        tr.stats.sampling_rate = 474505737
+        ds.add_waveforms(tr, tag="test")
+        dataset_name = ds.waveforms["AA.BB"].list()[0]
+    os.remove(asdf_filename)
+
+    assert dataset_name == (
+        "AA.BB..000__1971-03-28T03:19:05__1971-03-28T03:19:05__test")
+
+    # Check that leading nulls are also written.
+    with ASDFDataSet(asdf_filename) as ds:
+        tr.stats.starttime = obspy.UTCDateTime(0)
+        ds.add_waveforms(tr, tag="test")
+        dataset_name = ds.waveforms["AA.BB"].list()[0]
+    os.remove(asdf_filename)
+
+    assert dataset_name == (
+        "AA.BB..000__1970-01-01T00:00:00.000000000__"
+        "1970-01-01T00:00:00.000001635__test")

setup.py

@@ -15,6 +15,8 @@
 ::
     Version 0.4.x (TBD)
     -------------------
+    * Support for ASDF version 1.0.2. Allows writing traces that are less than
+      one second short (see #44, #45).
     * New get_waveform_attributes() method to quickly get all attributes
       for the waveforms of a stations (see #38, #39).