Automatically migrate TimeIntervals.timeseries to use TimeSeriesReferenceVectorData (#1390)

Co-authored-by: Ryan Ly <[email protected]>

Author: oruebel

CHANGELOG.md

@@ -4,10 +4,21 @@
 
 ### Breaking changes:
 - Restrict `SpatialSeries.data` to have no more than 3 columns (#1455)
+- Updated ``TimeIntervals`` to use the new  ``TimeSeriesReferenceVectorData`` type. This does not alter the overall
+  structure of ``TimeIntervals`` in a major way aside from changing the value of the ``neurodata_type`` attribute of the
+  ``TimeIntervals.timeseries`` column from ``VectorData`` to ``TimeSeriesReferenceVectorData``. This change facilitates
+  creating common functionality around ``TimeSeriesReferenceVectorData``. For NWB files with version 2.4.0 and earlier,
+  the ``TimeIntervals.timeseries`` column is automatically migrated on read in the ``TimeIntervalsMap``
+  object mapper class to use the ``TimeSeriesReferenceVectorData`` container class, so that users are presented a
+  consistent API for existing and new files. This change affects all existing ``TimeIntervals`` tables
+  e.g., ``NBWFile.epochs``, ``NWBFile.trials``, and ``NWBFile.invalid_times``. While this is technically a breaking
+  change, the impact user codes should be minimal as this change primarily adds functionality while the overall
+  behavior of the API is largely consistent with existing behavior. @oruebel, @rly (#1390)
 
 ### Documentation and tutorial enhancements:
+- Added tutorial on annotating data via ``TimeIntervals``. @oruebel (#1390)
 - Add copy button to code blocks @weiglszonja (#1460)
-- Create behavioral tutorial @weiglszonja (#1464) 
+- Create behavioral tutorial @weiglszonja (#1464)
 - Enhance display of icephys pandas tutorial by using ``dataframe_image`` to render and display large tables as images. @oruebel (#1469)
 - Create tutorial about reading and exploring an existing `NWBFile` @weiglszonja (#1453)
 

docs/gallery/general/file.py

@@ -486,6 +486,10 @@
 #
 # .. _basic_trials:
 #
+# The following provides a brief introduction to managing annotations in time via
+# :py:class:`~pynwb.epoch.TimeIntervals`. See the :ref:`time_intervals` tutorial
+# for a more detailed introduction to :py:class:`~pynwb.epoch.TimeIntervals`.
+#
 # Trials
 # ^^^^^^
 #

docs/gallery/general/plot_timeintervals.py

@@ -0,0 +1,242 @@
+"""
+.. _time_intervals:
+
+Annotating Time Intervals
+=========================
+
+Annotating events in time is a common need in neuroscience, e.g. to describes epochs, trials, and
+invalid times during an experimental session. NWB supports annotation of time intervals via the
+:py:class:`~pynwb.epoch.TimeIntervals` type. The :py:class:`~pynwb.epoch.TimeIntervals` type is
+a :py:class:`~hdmf.common.table.DynamicTable` with the following columns:
+
+1. :py:meth:`~pynwb.epoch.TimeIntervals.start_time` and :py:meth:`~pynwb.epoch.TimeIntervals.stop_time`
+   describe the start and stop times of intervals as floating point offsets in seconds relative to the
+   :py:meth:`~pynwb.file.NWBFile.timestamps_reference_time` of the file. In addition,
+2. :py:class:`~pynwb.epoch.TimeIntervals.tags` is an optional, indexed column used to associate user-defined string
+   tags with intervals (0 or more tags per time interval)
+3. :py:class:`~pynwb.epoch.TimeIntervals.timeseries` is an optional, indexed
+   :py:class:`~pynwb.base.TimeSeriesReferenceVectorData` column to map intervals directly to ranges in select,
+   relevant :py:class:`~pynwb.base.TimeSeries` (0 or more per time interval)
+4. as a :py:class:`~hdmf.common.table.DynamicTable` user may add additional columns to
+   :py:meth:`~pynwb.epoch.TimeIntervals` via :py:class:`~hdmf.common.table.DynamicTable.add_column`
+
+
+.. hint:: :py:meth:`~pynwb.epoch.TimeIntervals` is intended for storing general annotations of time ranges.
+          Depending on the application (e.g., when intervals are generated by data acquisition or automatic
+          data processing), it can be useful to describe intervals (or instantaneous events) in time
+          as :py:class:`~pynwb.base.TimeSeries`. NWB provides several types for this purposes, e.g.,
+          :py:class:`~pynwb.misc.IntervalSeries`, :py:class:`~pynwb.behavior.BehavioralEpochs`,
+          :py:class:`~pynwb.behavior.BehavioralEvents`, :py:class:`~pynwb.ecephys.EventDetection`, or
+          :py:class:`~pynwb.ecephys.SpikeEventSeries`.
+
+"""
+
+####################
+# Setup: Creating an example NWB file for the tutorial
+# ----------------------------------------------------
+#
+
+# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_timeintervals.png'
+from datetime import datetime
+from dateutil.tz import tzlocal
+from pynwb import NWBFile
+from pynwb import TimeSeries
+import numpy as np
+
+# create the NWBFile
+nwbfile = NWBFile(session_description='NWBFile to illustrate TimeIntervals basics',
+                  identifier='NWB123',
+                  session_start_time=datetime(2017, 4, 3, 11, tzinfo=tzlocal()),
+                  file_create_date=datetime(2017, 4, 15, 12, tzinfo=tzlocal()))
+# create some example TimeSeries
+test_ts = TimeSeries(name='series1',
+                     data=np.arange(1000),
+                     unit='m',
+                     timestamps=np.linspace(0.5, 601, 1000))
+rate_ts = TimeSeries(name='series2',
+                     data=np.arange(600),
+                     unit='V',
+                     starting_time=0.0, rate=1.0)
+# Add the TimeSeries to the file
+nwbfile.add_acquisition(test_ts)
+nwbfile.add_acquisition(rate_ts)
+
+####################
+# Adding Time Intervals to a NWBFile
+# ----------------------------------
+#
+# NWB provides a set of pre-defined :py:class:`~pynwb.epoch.TimeIntervals`
+# tables for :py:meth:`~pynwb.file.NWBFile.epochs`, :py:meth:`~pynwb.file.NWBFile.trials`, and
+# :py:meth:`~pynwb.file.NWBFile.invalid_times`.
+#
+# Trials
+# ^^^^^^
+#
+# Trials can be added to an NWB file using the methods :py:meth:`~pynwb.file.NWBFile.add_trial`
+# By default, NWBFile only requires trial :py:meth:`~pynwb.file.NWBFile.add_trial.start_time`
+# and :py:meth:`~pynwb.file.NWBFile.add_trial.end_time`. The :py:meth:`~pynwb.file.NWBFile.add_trial.tags`
+# and :py:meth:`~pynwb.file.NWBFile.add_trial.timeseries` are optional. For
+# :py:meth:`~pynwb.file.NWBFile.add_trial.timeseries` we only need to supply the :py:class:`~pynwb.base.TimeSeries`.
+# PyNWB automatically calculates the corresponding index range (described by ``idx_start`` and ``count``) for
+# the supplied  :py:class:`~pynwb.base.TimeSeries based on the given ``start_time`` and ``stop_time`` and
+# the :py:meth:`~pynwb.base.TimeSeries.timestamps` (or :py:class:`~pynwb.base.TimeSeries.starting_time`
+# and :py:meth:`~pynwb.base.TimeSeries.rate`) of the given :py:class:`~pynwb.base.TimeSeries`.
+#
+# Additional columns can be added using :py:meth:`~pynwb.file.NWBFile.add_trial_column`. This method takes a name
+# for the column and a description of what the column stores. You do not need to supply data
+# type, as this will inferred. Once all columns have been added, trial data can be populated using
+# :py:meth:`~pynwb.file.NWBFile.add_trial`.
+#
+# Lets add an additional column and some trial data with tags and timeseries references.
+
+nwbfile.add_trial_column(name='stim', description='the visual stimuli during the trial')
+
+nwbfile.add_trial(start_time=0.0, stop_time=2.0, stim='dog',
+                  tags=['animal'], timeseries=[test_ts, rate_ts])
+nwbfile.add_trial(start_time=3.0, stop_time=5.0, stim='mountain',
+                  tags=['landscape'], timeseries=[test_ts, rate_ts])
+nwbfile.add_trial(start_time=6.0, stop_time=8.0, stim='desert',
+                  tags=['landscape'], timeseries=[test_ts, rate_ts])
+nwbfile.add_trial(start_time=9.0, stop_time=11.0, stim='tree',
+                  tags=['landscape', 'plant'], timeseries=[test_ts, rate_ts])
+nwbfile.add_trial(start_time=12.0, stop_time=14.0, stim='bird',
+                  tags=['animal'], timeseries=[test_ts, rate_ts])
+nwbfile.add_trial(start_time=15.0, stop_time=17.0, stim='flower',
+                  tags=['animal'], timeseries=[test_ts, rate_ts])
+
+####################
+# Epochs
+# ^^^^^^
+#
+# Similarly, epochs can be added to an NWB file using the method :py:meth:`~pynwb.file.NWBFile.add_epoch` and
+# :py:meth:`~pynwb.file.NWBFile.add_epoch_column`.
+
+nwbfile.add_epoch(2.0, 4.0, ['first', 'example'], [test_ts, ])
+nwbfile.add_epoch(6.0, 8.0, ['second', 'example'], [test_ts, ])
+
+####################
+# Invalid Times
+# ^^^^^^^^^^^^^
+#
+# Similarly, invalid times can be added using the method :py:meth:`~pynwb.file.NWBFile.add_invalid_time_interval` and
+# :py:meth:`~pynwb.file.NWBFile.add_invalid_times_column`.
+
+nwbfile.add_epoch(2.0, 4.0, ['first', 'example'], [test_ts, ])
+nwbfile.add_epoch(6.0, 8.0, ['second', 'example'], [test_ts, ])
+
+####################
+# Custom Time Intervals
+# ^^^^^^^^^^^^^^^^^^^^^
+#
+# To define custom, experiment-specific :py:class:`~pynwb.epoch.TimeIntervals` we can wither add them
+# either: 1) when creating the :py:class:`~pynwb.file.NWBFile` by defining the
+# :py:meth:`~pynwb.file.NWBFile.__init__.intervals` constructor argument or 2) via the
+# :py:meth:`~pynwb.file.NWBFile.add_time_intervals` or :py:meth:`~pynwb.file.NWBFile.create_time_intervals`
+# after the :py:class:`~pynwb.file.NWBFile` has been created.
+#
+
+from pynwb.epoch import TimeIntervals
+
+sleep_stages = TimeIntervals(
+    name="sleep_stages",
+    description="intervals for each sleep stage as determined by EEG",
+)
+
+sleep_stages.add_column(name="stage", description="stage of sleep")
+sleep_stages.add_column(name="confidence", description="confidence in stage (0-1)")
+
+sleep_stages.add_row(start_time=0.3, stop_time=0.5, stage=1, confidence=.5)
+sleep_stages.add_row(start_time=0.7, stop_time=0.9, stage=2, confidence=.99)
+sleep_stages.add_row(start_time=1.3, stop_time=3.0, stage=3, confidence=0.7)
+
+_ = nwbfile.add_time_intervals(sleep_stages)
+
+
+####################
+# Accessing Time Intervals
+# ------------------------
+#
+# We can access the predefined :py:class:`~pynwb.epoch.TimeIntervals` tables via the corresponding
+# :py:meth:`~pynwb.file.NWBFile.epochs`, :py:meth:`~pynwb.file.NWBFile.trials`, and
+# :py:meth:`~pynwb.file.NWBFile.invalid_times` properties and for custom :py:class:`~pynwb.epoch.TimeIntervals`
+# via the :py:meth:`~pynwb.file.NWBFile.get_time_intervals`  method. E.g.:
+
+_ = nwbfile.intervals
+_ = nwbfile.get_time_intervals('sleep_stages')
+
+
+####################
+# Like any :py:class:`~hdmf.common.table.DynamicTable`, we can conveniently convert any
+# :py:class:`~pynwb.epoch.TimeIntervals` table to a ``pandas.DataFrame`` via
+# :py:meth:`~hdmf.common.table.DynamicTable.to_dataframe`, such as:
+
+nwbfile.trials.to_dataframe()
+
+####################
+# This approach makes it easy to query the data to, e.g., locate all time intervals within a certain time range
+
+trials_df = nwbfile.trials.to_dataframe()
+trials_df.query('(start_time > 2.0) & (stop_time < 9.0)')
+
+####################
+# Accessing referenced TimeSeries
+# -------------------------------
+#
+# As mentioned earlier, the ``timeseries`` column is defined by a :py:class:`~pynwb.base.TimeSeriesReferenceVectorData`
+# which stores references to the corresponding ranges in :py:class:`~pynwb.base.TimeSeries`. Individual references
+# to :py:class:`~pynwb.base.TimeSeries` are described via :py:class:`~pynwb.base.TimeSeriesReference` tuples
+# with the :py:class:`~pynwb.base.TimeSeriesReference.idx_start`, :py:class:`~pynwb.base.TimeSeriesReference.count`,
+# and :py:class:`~pynwb.base.TimeSeriesReference.timeseries`.
+# Using :py:class:`~pynwb.base.TimeSeriesReference` we can easily access the relevant
+# :py:meth:`~pynwb.base.TimeSeriesReference.data` and :py:meth:`~pynwb.base.TimeSeriesReference.timestamps`
+# for the corresponding time range from the :py:class:`~pynwb.base.TimeSeries`.
+
+# Get a single example TimeSeriesReference from the trials table
+example_tsr = nwbfile.trials['timeseries'][0][0]
+
+# Get the data values from the timeseries. This is a shorthand for:
+# _ = example_tsr.timeseries.data[example_tsr.idx_start: (example_tsr.idx_start + example_tsr.count)]
+_ = example_tsr.data
+
+# Get the timestamps. Timestamps are either loaded from the TimeSeries or
+# computed from the starting_time and rate
+example_tsr.timestamps
+
+####################
+# Using :py:class:`~pynwb.base.TimeSeriesReference.isvalid` we can further check if the reference is valid.
+# A :py:class:`~pynwb.base.TimeSeriesReference` is defined as invalid if both
+# :py:class:`~pynwb.base.TimeSeriesReference.idx_start`, :py:class:`~pynwb.base.TimeSeriesReference.count` are
+# set to ``-1``. :py:class:`~pynwb.base.TimeSeriesReference.isvalid` further also checks that the indicated
+# index range and types are valid, raising ``IndexError`` and ``TypeError`` respectively, if bad
+# :py:class:`~pynwb.base.TimeSeriesReference.idx_start`, :py:class:`~pynwb.base.TimeSeriesReference.count` or
+# :py:class:`~pynwb.base.TimeSeriesReference.timeseries` are found.
+
+example_tsr.isvalid()
+
+####################
+# Adding TimeSeries references to other tables
+# --------------------------------------------
+#
+# Since :py:class:`~pynwb.base.TimeSeriesReferenceVectorData` is a regular :py:class:`~hdmf.common.table.VectoData`
+# type, we can use it to add references to intervals in :py:class:`~pynwb.base.TimeSeries` to any
+# :py:class:`~hdmf.common.table.DynamicTable`. In the :py:class:`~pynwb.icephys.IntracellularRecordingTable`, e.g.,
+# it is used to reference the recording of the stimulus and response associated with a particular intracellular
+# electrophysiology recording.
+#
+
+
+####################
+# Reading/Writing TimeIntervals to file
+# -------------------------------------
+#
+# Reading and writing the data is as usual:
+
+from pynwb import NWBHDF5IO
+# write the file
+with NWBHDF5IO('example_timeintervals_file.nwb', 'w') as io:
+    io.write(nwbfile)
+# read the file
+io = NWBHDF5IO('example_timeintervals_file.nwb', 'r')
+nwbfile_in = io.read()
+# plot the sleep stages TimeIntervals table
+nwbfile.get_time_intervals('sleep_stages').to_dataframe()

docs/source/figures/gallery_thumbnails.pptx

@@ -485,18 +485,38 @@ class TimeSeriesReferenceVectorData(VectorData):
             *get_docval(VectorData.__init__, 'data'))
     def __init__(self, **kwargs):
         call_docval_func(super().__init__, kwargs)
+        # CAUTION: Define any logic specific for init in the self._init_internal function, not here!
+        self._init_internal()
 
-    @docval({'name': 'val', 'type': TIME_SERIES_REFERENCE_TUPLE, 'doc': 'the value to add to this column'})
+    def _init_internal(self):
+        """
+        Called from __init__ to perform initialization specific to this class. This is done
+        here due to the :py:class:`~pynwb.io.epoch.TimeIntervalsMap` having to migrate legacy VectorData
+        to TimeSeriesReferenceVectorData. In this way, if dedicated logic init logic needs
+        to be added to this class then we have a place for it without having to also
+        update :py:class:`~pynwb.io.epoch.TimeIntervalsMap` (which would likely get forgotten)
+        """
+        pass
+
+    @docval({'name': 'val', 'type': (TIME_SERIES_REFERENCE_TUPLE, tuple),
+             'doc': 'the value to add to this column. If this is a regular tuple then it '
+                    'must be convertible to a TimeSeriesReference'})
     def add_row(self, **kwargs):
         """Append a data value to this column."""
         val = getargs('val', kwargs)
+        if not (isinstance(val, self.TIME_SERIES_REFERENCE_TUPLE)):
+            val = self.TIME_SERIES_REFERENCE_TUPLE(*val)
         val.check_types()
         super().append(val)
 
-    @docval({'name': 'arg', 'type': TIME_SERIES_REFERENCE_TUPLE, 'doc': 'the value to append to this column'})
+    @docval({'name': 'arg', 'type': (TIME_SERIES_REFERENCE_TUPLE, tuple),
+             'doc': 'the value to append to this column. If this is a regular tuple then it '
+                    'must be convertible to a TimeSeriesReference'})
     def append(self, **kwargs):
         """Append a data value to this column."""
         arg = getargs('arg', kwargs)
+        if not (isinstance(arg, self.TIME_SERIES_REFERENCE_TUPLE)):
+            arg = self.TIME_SERIES_REFERENCE_TUPLE(*arg)
         arg.check_types()
         super().append(arg)
 
@@ -535,7 +555,7 @@ def get(self, key, **kwargs):
                 return self.TIME_SERIES_REFERENCE_TUPLE(*vals)
         else:  # key selected multiple rows
             # When loading from HDF5 we get an np.ndarray otherwise we get list-of-list. This
-            # makes the values consistent and tranforms the data to use our namedtuple type
+            # makes the values consistent and transforms the data to use our namedtuple type
             re = [self.TIME_SERIES_REFERENCE_NONE_TYPE
                   if (v[0] < 0 or v[1] < 0) else self.TIME_SERIES_REFERENCE_TUPLE(*v)
                   for v in vals]

docs/source/figures/gallery_thumbnails_timeintervals.png

@@ -4,7 +4,7 @@
 from hdmf.data_utils import DataIO
 
 from . import register_class, CORE_NAMESPACE
-from .base import TimeSeries
+from .base import TimeSeries, TimeSeriesReferenceVectorData, TimeSeriesReference
 from hdmf.common import DynamicTable
 
 
@@ -20,7 +20,8 @@ class TimeIntervals(DynamicTable):
         {'name': 'start_time', 'description': 'Start time of epoch, in seconds', 'required': True},
         {'name': 'stop_time', 'description': 'Stop time of epoch, in seconds', 'required': True},
         {'name': 'tags', 'description': 'user-defined tags', 'index': True},
-        {'name': 'timeseries', 'description': 'index into a TimeSeries object', 'index': True}
+        {'name': 'timeseries', 'description': 'index into a TimeSeries object',
+         'index': True, 'class': TimeSeriesReferenceVectorData}
     )
 
     @docval({'name': 'name', 'type': str, 'doc': 'name of this TimeIntervals'},  # required
@@ -51,7 +52,7 @@ def add_interval(self, **kwargs):
             tmp = list()
             for ts in timeseries:
                 idx_start, count = self.__calculate_idx_count(start_time, stop_time, ts)
-                tmp.append((idx_start, count, ts))
+                tmp.append(TimeSeriesReference(idx_start, count, ts))
             timeseries = tmp
             rkwargs['timeseries'] = timeseries
         return super(TimeIntervals, self).add_row(**rkwargs)