Merge pull request #1544 from samuelgarcia/add_signal_buffer_id

Simple proposal for `buffer_id` in rawio

Author: zm711

neo/rawio/alphaomegarawio.py

@@ -35,6 +35,7 @@
 
 from .baserawio import (
     BaseRawIO,
+    _signal_buffer_dtype,
     _signal_stream_dtype,
     _signal_channel_dtype,
     _spike_channel_dtype,
@@ -652,8 +653,9 @@ def _parse_header(self):
         # could be loaded in any order
         self._merge_segments()
 
+        buffer_id = ""
         signal_streams = set(
-            (stream_name, stream_id)
+            (stream_name, stream_id, buffer_id)
             for segment in self._segments
             for stream in segment["streams"]
             for stream_name, _, stream_id in self.STREAM_CHANNELS
@@ -662,7 +664,9 @@ def _parse_header(self):
         signal_streams = list(signal_streams)
         signal_streams.sort(key=lambda x: x[1])
         signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)
+        signal_buffers = np.array([], dtype=_signal_buffer_dtype)
 
+        buffer_id = ""
         signal_channels = set(
             (
                 channel["name"],
@@ -672,11 +676,12 @@ def _parse_header(self):
                 "uV",
                 channel["gain"] / channel["bit_resolution"],
                 0,
-                stream,
+                stream_id,
+                buffer_id,
             )
             for segment in self._segments
-            for stream in segment["streams"]
-            for channel_id, channel in segment["streams"][stream].items()
+            for stream_id in segment["streams"]
+            for channel_id, channel in segment["streams"][stream_id].items()
         )
         signal_channels = list(signal_channels)
         signal_channels.sort(key=lambda x: (x[7], x[0]))
@@ -709,6 +714,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [len(self._segments)]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = signal_channels
         self.header["spike_channels"] = spike_channels

neo/rawio/axographrawio.py

@@ -158,6 +158,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -811,7 +812,9 @@ def _scan_axograph_file(self):
                     self.logger.debug("initial data: {array[:5] * gain + offset}")
 
                     # channel_info will be cast to _signal_channel_dtype
-                    channel_info = (name, str(i), 1 / sampling_period, f.byte_order + dtype, units, gain, offset, "0")
+                    buffer_id = ""
+                    stream_id = "0"
+                    channel_info = (name, str(i), 1 / sampling_period, f.byte_order + dtype, units, gain, offset, stream_id, buffer_id)
 
                     self.logger.debug("channel_info: {channel_info}")
                     self.logger.debug("")
@@ -1230,13 +1233,15 @@ def _scan_axograph_file(self):
         event_channels.append(("AxoGraph Intervals", "", "epoch"))
 
         if len(sig_channels) > 0:
-            signal_streams = [("Signals", "0")]
+            signal_streams = [("Signals", "0", "")]
         else:
             signal_streams = []
+        signal_buffers = []
 
         # organize header
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [1]
+        self.header["signal_buffers"] = np.array(signal_buffers, dtype=_signal_buffer_dtype)
         self.header["signal_streams"] = np.array(signal_streams, dtype=_signal_stream_dtype)
         self.header["signal_channels"] = np.array(sig_channels, dtype=_signal_channel_dtype)
         self.header["event_channels"] = np.array(event_channels, dtype=_event_channel_dtype)

neo/rawio/axonarawio.py

@@ -25,6 +25,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -199,6 +200,7 @@ def _parse_header(self):
         params["set"]["sampling_rate"] = int(set_dict["rawRate"])
 
         # SCAN BIN FILE
+        signal_buffers = []
         signal_streams = []
         signal_channels = []
         if self.bin_file:
@@ -289,6 +291,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [1]
+        self.header["signal_buffers"] = np.array(signal_buffers, dtype=_signal_buffer_dtype)
         self.header["signal_streams"] = np.array(signal_streams, dtype=_signal_stream_dtype)
         self.header["signal_channels"] = np.array(signal_channels, dtype=_signal_channel_dtype)
         self.header["spike_channels"] = np.array(spike_channels, dtype=_spike_channel_dtype)
@@ -315,7 +318,7 @@ def _parse_header(self):
 
     def _get_signal_streams_header(self):
         # create signals stream information (we always expect a single stream)
-        return np.array([("stream 0", "0")], dtype=_signal_stream_dtype)
+        return np.array([("stream 0", "0", "")], dtype=_signal_stream_dtype)
 
     def _segment_t_start(self, block_index, seg_index):
         return 0.0
@@ -638,10 +641,11 @@ def _get_signal_chan_header(self):
                 chan_id = str(cntr)
                 gain = gain_list[cntr]
                 stream_id = "0"
+                buffer_id = ""
                 # the sampling rate information is stored in the set header
                 # and not in the bin file
                 sr = self.file_parameters["set"]["sampling_rate"]
-                sig_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id))
+                sig_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id, buffer_id))
 
         return np.array(sig_channels, dtype=_signal_channel_dtype)
 

neo/rawio/axonrawio.py

@@ -56,6 +56,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -227,12 +228,14 @@ def _parse_header(self):
             else:
                 gain, offset = 1.0, 0.0
             stream_id = "0"
-            signal_channels.append((name, str(chan_id), self._sampling_rate, sig_dtype, units, gain, offset, stream_id))
+            buffer_id = "0"
+            signal_channels.append((name, str(chan_id), self._sampling_rate, sig_dtype, units, gain, offset, stream_id, buffer_id))
 
         signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype)
 
-        # one unique signal stream
-        signal_streams = np.array([("Signals", "0")], dtype=_signal_stream_dtype)
+        # one unique signal stream and buffer
+        signal_buffers = np.array([("Signals", "0")], dtype=_signal_buffer_dtype)
+        signal_streams = np.array([("Signals", "0", "0")], dtype=_signal_stream_dtype)
 
         # only one events channel : tag
         # In ABF timstamps are not attached too any particular segment
@@ -258,6 +261,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [nb_segment]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = signal_channels
         self.header["spike_channels"] = spike_channels

neo/rawio/baserawio.py

@@ -86,9 +86,15 @@
 
 error_header = "Header is not read yet, do parse_header() first"
 
+_signal_buffer_dtype = [
+    ("name", "U64"),  # not necessarily unique
+    ("id", "U64"),  # must be unique
+]
+# To be left an empty array if the concept of buffer is undefined for a reader.
 _signal_stream_dtype = [
     ("name", "U64"),  # not necessarily unique
     ("id", "U64"),  # must be unique
+    ("buffer_id", "U64"), # should be "" (empty string) when the stream is not nested under a buffer or the buffer is undefined for some reason.
 ]
 
 _signal_channel_dtype = [
@@ -100,6 +106,7 @@
     ("gain", "float64"),
     ("offset", "float64"),
     ("stream_id", "U64"),
+    ("buffer_id", "U64"),
 ]
 
 # TODO for later: add t_start and length in _signal_channel_dtype

neo/rawio/bci2000rawio.py

@@ -15,6 +15,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -49,8 +50,10 @@ def _parse_header(self):
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [1]
 
-        # one unique stream
-        signal_streams = np.array([("Signals", "0")], dtype=_signal_stream_dtype)
+        # one unique stream and buffer
+        signal_buffers = np.array(("Signals", "0"), dtype=_signal_buffer_dtype)
+        signal_streams = np.array([("Signals", "0", "0")], dtype=_signal_stream_dtype)
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
 
         sig_channels = []
@@ -78,7 +81,8 @@ def _parse_header(self):
                 offset = float(offset)
 
             stream_id = "0"
-            sig_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id))
+            buffer_id = "0"
+            sig_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id, buffer_id))
         self.header["signal_channels"] = np.array(sig_channels, dtype=_signal_channel_dtype)
 
         self.header["spike_channels"] = np.array([], dtype=_spike_channel_dtype)

neo/rawio/biocamrawio.py

@@ -14,6 +14,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -66,7 +67,9 @@ def _parse_header(self):
         gain = self._header_dict["gain"]
         offset = self._header_dict["offset"]
 
-        signal_streams = np.array([("Signals", "0")], dtype=_signal_stream_dtype)
+        # buffer concept cannot be used in this reader because of too complicated dtype across version
+        signal_buffers = np.array([], dtype=_signal_stream_dtype)
+        signal_streams = np.array([("Signals", "0", "")], dtype=_signal_stream_dtype)
 
         sig_channels = []
         for c, chan in enumerate(self._channels):
@@ -78,7 +81,8 @@ def _parse_header(self):
             gain = gain
             offset = offset
             stream_id = "0"
-            sig_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id))
+            buffer_id = ""
+            sig_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id, buffer_id))
         sig_channels = np.array(sig_channels, dtype=_signal_channel_dtype)
 
         # No events
@@ -92,6 +96,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [1]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = sig_channels
         self.header["spike_channels"] = spike_channels

neo/rawio/blackrockrawio.py

@@ -72,6 +72,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -253,6 +254,7 @@ def _parse_header(self):
 
         event_channels = []
         spike_channels = []
+        signal_buffers = []
         signal_streams = []
         signal_channels = []
 
@@ -399,7 +401,11 @@ def _parse_header(self):
                         ext_header.append(d)
 
                 if len(ext_header) > 0:
-                    signal_streams.append((f"nsx{nsx_nb}", str(nsx_nb)))
+                    # in blackrock : one stream per buffer so same id
+                    buffer_id = stream_id = str(nsx_nb)
+                    stream_name = f"nsx{nsx_nb}"
+                    signal_buffers.append((stream_name, buffer_id))
+                    signal_streams.append((stream_name, stream_id, buffer_id))
                 for i, chan in enumerate(ext_header):
                     if spec in ["2.2", "2.3", "3.0"]:
                         ch_name = chan["electrode_label"].decode()
@@ -420,8 +426,8 @@ def _parse_header(self):
                             float(chan["max_digital_val"]) - float(chan["min_digital_val"])
                         )
                         offset = -float(chan["min_digital_val"]) * gain + float(chan["min_analog_val"])
-                    stream_id = str(nsx_nb)
-                    signal_channels.append((ch_name, ch_id, sr, sig_dtype, units, gain, offset, stream_id))
+                    buffer_id = stream_id = str(nsx_nb)
+                    signal_channels.append((ch_name, ch_id, sr, sig_dtype, units, gain, offset, stream_id, buffer_id))
 
             # check nb segment per nsx
             nb_segments_for_nsx = [len(self.nsx_datas[nsx_nb]) for nsx_nb in self.nsx_to_load]
@@ -509,10 +515,12 @@ def _parse_header(self):
         event_channels = np.array(event_channels, dtype=_event_channel_dtype)
         signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype)
         signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)
+        signal_buffers = np.array(signal_buffers, dtype=_signal_buffer_dtype)
 
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [self._nb_segment]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = signal_channels
         self.header["spike_channels"] = spike_channels

neo/rawio/brainvisionrawio.py

@@ -16,6 +16,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -83,7 +84,8 @@ def _parse_header(self):
             sigs = sigs[: -sigs.size % nb_channel]
         self._raw_signals = sigs.reshape(-1, nb_channel)
 
-        signal_streams = np.array([("Signals", "0")], dtype=_signal_stream_dtype)
+        signal_buffers = np.array(("Signals", "0"), dtype=_signal_buffer_dtype)
+        signal_streams = np.array([("Signals", "0", "0")], dtype=_signal_stream_dtype)
 
         sig_channels = []
         channel_infos = vhdr_header["Channel Infos"]
@@ -115,7 +117,8 @@ def _parse_header(self):
                 gain = 1
             offset = 0
             stream_id = "0"
-            sig_channels.append((name, chan_id, self._sampling_rate, sig_dtype, units, gain, offset, stream_id))
+            buffer_id = "0"
+            sig_channels.append((name, chan_id, self._sampling_rate, sig_dtype, units, gain, offset, stream_id, buffer_id))
         sig_channels = np.array(sig_channels, dtype=_signal_channel_dtype)
 
         # No spikes
@@ -152,6 +155,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [1]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = sig_channels
         self.header["spike_channels"] = spike_channels

neo/rawio/cedrawio.py

@@ -26,6 +26,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -102,7 +103,8 @@ def _parse_header(self):
                 dtype = "int16"
                 # set later after grouping
                 stream_id = "0"
-                signal_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id))
+                buffer_id = ""
+                signal_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id, buffer_id))
 
             elif chan_type == sonpy.lib.DataType.AdcMark:
                 # spike and waveforms : only spike times is used here
@@ -142,8 +144,11 @@ def _parse_header(self):
             signal_channels["stream_id"][mask] = stream_id
             num_chans = np.sum(mask)
             stream_name = f"{stream_id} {num_chans}chans"
-            signal_streams.append((stream_name, stream_id))
+            buffer_id = ""
+            signal_streams.append((stream_name, stream_id, buffer_id))
         signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)
+        # buffer is unknown because using a close source API
+        signal_buffers = np.array([], dtype=_signal_buffer_dtype)
 
         # spike channels not handled
         spike_channels = np.array(spike_channels, dtype=_spike_channel_dtype)
@@ -162,6 +167,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [1]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = signal_channels
         self.header["spike_channels"] = spike_channels