wip : more rawio with buffer_ids

Author: samuelgarcia

neo/rawio/neuroscoperawio.py

@@ -25,6 +25,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -107,8 +108,9 @@ def _parse_header(self):
         # Extract signal from the data file
         self._raw_signals = np.memmap(self.data_file_path, dtype=sig_dtype, mode="r", offset=0).reshape(-1, nb_channel)
 
-        # one unique stream
-        signal_streams = np.array([("Signals", "0")], dtype=_signal_stream_dtype)
+        # one unique stream and buffer
+        signal_buffers = np.array([("Signals", "0")], dtype=_signal_buffer_dtype)
+        signal_streams = np.array([("Signals", "0", "0")], dtype=_signal_stream_dtype)
 
         # signals
         sig_channels = []
@@ -118,7 +120,8 @@ def _parse_header(self):
             units = "mV"
             offset = 0.0
             stream_id = "0"
-            sig_channels.append((name, chan_id, self._sampling_rate, sig_dtype, units, gain, offset, stream_id))
+            buffer_id = "0"
+            sig_channels.append((name, chan_id, self._sampling_rate, sig_dtype, units, gain, offset, stream_id, buffer_id))
         sig_channels = np.array(sig_channels, dtype=_signal_channel_dtype)
 
         # No events
@@ -133,6 +136,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [1]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = sig_channels
         self.header["spike_channels"] = spike_channels

neo/rawio/nixrawio.py

@@ -13,6 +13,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -77,7 +78,8 @@ def _parse_header(self):
                             stream_ids.append(stream_id)
                         gain = 1
                         offset = 0.0
-                        signal_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id))
+                        buffer_id = ""
+                        signal_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id, buffer_id))
                 # only read structure of first segment and assume the same
                 # across segments
                 break
@@ -86,6 +88,9 @@ def _parse_header(self):
         signal_streams = np.zeros(len(stream_ids), dtype=_signal_stream_dtype)
         signal_streams["id"] = stream_ids
         signal_streams["name"] = ""
+        signal_streams["buffer_id"] = ""
+        # potentially nix support buffer but because we use the python module API and not h5py then no buffer
+        signal_buffers = np.array([], dtype=_signal_buffer_dtype)
 
         spike_channels = []
         unit_name = ""
@@ -194,6 +199,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = len(self.file.blocks)
         self.header["nb_segment"] = [len(seg_groups) for bl in self.file.blocks]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = signal_channels
         self.header["spike_channels"] = spike_channels

neo/rawio/openephysbinaryrawio.py

@@ -19,6 +19,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -120,6 +121,7 @@ def _parse_header(self):
         for stream_index, stream_name in enumerate(sig_stream_names):
             # stream_index is the index in vector sytream names
             stream_id = str(stream_index)
+            buffer_id = stream_id
             info = self._sig_streams[0][0][stream_index]
             new_channels = []
             for chan_info in info["channels"]:
@@ -141,16 +143,21 @@ def _parse_header(self):
                         chan_info["bit_volts"],
                         0.0,
                         stream_id,
+                        buffer_id,
                     )
                 )
             signal_channels.extend(new_channels)
         signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype)
 
         signal_streams = []
+        signal_buffers = []
         for stream_index, stream_name in enumerate(sig_stream_names):
             stream_id = str(stream_index)
-            signal_streams.append((stream_name, stream_id))
+            buffer_id = str(stream_index)
+            signal_buffers.append((stream_name, buffer_id))
+            signal_streams.append((stream_name, stream_id, buffer_id))
         signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)
+        signal_buffers = np.array(signal_buffers, dtype=_signal_buffer_dtype)
 
         # create memmap for signals
         for block_index in range(nb_block):
@@ -299,6 +306,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = nb_block
         self.header["nb_segment"] = nb_segment_per_block
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = signal_channels
         self.header["spike_channels"] = spike_channels

neo/rawio/openephysrawio.py

@@ -17,6 +17,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -162,6 +163,8 @@ def _parse_header(self):
                         units = "uV"
                     else:
                         units = "V"
+                    stream_id = processor_id
+                    buffer_id = ""
                     signal_channels.append(
                         (
                             ch_name,
@@ -171,7 +174,8 @@ def _parse_header(self):
                             units,
                             chan_info["bitVolts"],
                             0.0,
-                            processor_id,
+                            stream_id,
+                            buffer_id,
                         )
                     )
 
@@ -221,10 +225,13 @@ def _parse_header(self):
             # and create streams channels (keep natural order 'CH' first)
             stream_ids, order = np.unique(chan_stream_ids, return_index=True)
             stream_ids = stream_ids[np.argsort(order)]
-            signal_streams = [(f"Signals {stream_id}", f"{stream_id}") for stream_id in stream_ids]
-            signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)
+            signal_streams = [(f"Signals {stream_id}", f"{stream_id}", "") for stream_id in stream_ids]
         else:
-            signal_streams = np.array([])
+            signal_streams = []
+        signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)
+        # no buffer handling in this format because one channel per file
+        signal_buffers = np.array( [], dtype=_signal_buffer_dtype)
+
         # scan for spikes files
         spike_channels = []
 
@@ -328,6 +335,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [nb_segment]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = signal_channels
         self.header["spike_channels"] = spike_channels

neo/rawio/phyrawio.py

@@ -12,6 +12,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -114,8 +115,8 @@ def _parse_header(self):
         self._t_start = 0.0
         self._t_stop = max(self._spike_times).item() / self._sampling_frequency
 
-        signal_streams = []
-        signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)
+        signal_streams = np.array([], dtype=_signal_stream_dtype)
+        signal_buffers = np.array([], dtype=_signal_buffer_dtype)
 
         signal_channels = []
         signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype)
@@ -138,6 +139,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [1]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = signal_channels
         self.header["spike_channels"] = spike_channels

neo/rawio/plexonrawio.py

@@ -39,6 +39,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -259,10 +260,14 @@ def _parse_header(self):
                 )
             offset = 0.0
             stream_id = "0"  # This is overwritten later
-            sig_channels.append((name, str(chan_id), sampling_rate, sig_dtype, units, gain, offset, stream_id))
+            buffer_id = ""
+            sig_channels.append((name, str(chan_id), sampling_rate, sig_dtype, units, gain, offset, stream_id, buffer_id))
 
         sig_channels = np.array(sig_channels, dtype=_signal_channel_dtype)
 
+        # no buffer here because block are splitted by channel
+        signal_buffers = np.array([], dtype=_signal_buffer_dtype)
+
         if sig_channels.size == 0:
             signal_streams = np.array([], dtype=_signal_stream_dtype)
 
@@ -309,7 +314,8 @@ def _parse_header(self):
                 self._sig_sampling_rate[stream_index] = sr
                 self._signal_length[stream_index] = length
 
-                signal_streams.append((stream_name, stream_id))
+                buffer_id = ""
+                signal_streams.append((stream_name, stream_id, buffer_id))
 
             signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)
 
@@ -373,6 +379,7 @@ def _parse_header(self):
         self.header = {
             "nb_block": 1,
             "nb_segment": [1],
+            "signal_buffers": signal_buffers,
             "signal_streams": signal_streams,
             "signal_channels": sig_channels,
             "spike_channels": spike_channels,

neo/rawio/rawbinarysignalrawio.py

@@ -25,6 +25,7 @@ class RawBinarySignalIO
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -93,6 +94,7 @@ def _parse_header(self):
                 chan_id = f"{c}"
                 units = ""
                 stream_id = "0"
+                buffer_id = "0"
                 signal_channels.append(
                     (
                         name,
@@ -103,14 +105,16 @@ def _parse_header(self):
                         self.signal_gain,
                         self.signal_offset,
                         stream_id,
+                        buffer_id
                     )
                 )
 
         signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype)
 
-        # one unique stream
+        # one unique stream and buffer
         if signal_channels.size > 0:
-            signal_streams = np.array([("Signals", "0")], dtype=_signal_stream_dtype)
+            signal_buffers = np.array([("Signals", "0")], dtype=_signal_buffer_dtype)
+            signal_streams = np.array([("Signals", "0", "0")], dtype=_signal_stream_dtype)
         else:
             signal_streams = np.array([], dtype=_signal_stream_dtype)
 
@@ -126,6 +130,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [1]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = signal_channels
         self.header["spike_channels"] = spike_channels

neo/rawio/rawmcsrawio.py

@@ -20,6 +20,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -53,8 +54,9 @@ def _parse_header(self):
         self.sampling_rate = info["sampling_rate"]
         self.nb_channel = len(info["channel_names"])
 
-        # one unique stream
-        signal_streams = np.array([("Signals", "0")], dtype=_signal_stream_dtype)
+        # one unique stream and buffer
+        signal_streams = np.array([("Signals", "0", "0")], dtype=_signal_stream_dtype)
+        signal_buffers = np.array([("Signals", "0")], dtype=_signal_buffer_dtype)
 
         self._raw_signals = np.memmap(self.filename, dtype=self.dtype, mode="r", offset=info["header_size"]).reshape(
             -1, self.nb_channel
@@ -64,6 +66,7 @@ def _parse_header(self):
         for c in range(self.nb_channel):
             chan_id = str(c)
             stream_id = "0"
+            buffer_id = "0"
             sig_channels.append(
                 (
                     info["channel_names"][c],
@@ -74,6 +77,7 @@ def _parse_header(self):
                     info["signal_gain"],
                     info["signal_offset"],
                     stream_id,
+                    buffer_id,
                 )
             )
         sig_channels = np.array(sig_channels, dtype=_signal_channel_dtype)
@@ -90,6 +94,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [1]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = sig_channels
         self.header["spike_channels"] = spike_channels

neo/rawio/spike2rawio.py

@@ -26,6 +26,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -254,7 +255,8 @@ def _parse_header(self):
                     offset = 0.0
                     sig_dtype = "float32"
                 stream_id = "0"  # set it after the loop
-                signal_channels.append((name, str(chan_id), sampling_rate, sig_dtype, units, gain, offset, stream_id))
+                buffer_id = ""
+                signal_channels.append((name, str(chan_id), sampling_rate, sig_dtype, units, gain, offset, stream_id, buffer_id))
 
             elif chan_info["kind"] in [2, 3, 4, 5, 8]:
                 # Event
@@ -297,6 +299,8 @@ def _parse_header(self):
         spike_channels = np.array(spike_channels, dtype=_spike_channel_dtype)
         event_channels = np.array(event_channels, dtype=_event_channel_dtype)
 
+        signal_buffers = np.array([], dtype=_signal_buffer_dtype)
+
         if len(signal_channels) > 0:
             if self.try_signal_grouping:
                 # try to group signals channel if same sampling_rate/dtype/...
@@ -324,7 +328,8 @@ def _parse_header(self):
                                 "Signal channel in groups do not have same size, use try_signal_grouping=False"
                             )
                     self._sig_dtypes[stream_id] = np.dtype(charact["dtype"])
-                    signal_streams.append((f"Signal stream {stream_id}", stream_id))
+                    buffer_id = ""
+                    signal_streams.append((f"Signal stream {stream_id}", stream_id, buffer_id))
                 signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)
             else:
                 # if try_signal_grouping fail the user can try to split each channel in
@@ -333,6 +338,7 @@ def _parse_header(self):
                 signal_streams = np.zeros(signal_channels.size, dtype=_signal_stream_dtype)
                 signal_streams["id"] = signal_channels["stream_id"]
                 signal_streams["name"] = signal_channels["name"]
+                signal_streams["buffer_id"] = ""
                 self._sig_dtypes = {s["stream_id"]: np.dtype(s["dtype"]) for s in signal_channels}
         else:
             signal_streams = np.array([], dtype=_signal_stream_dtype)
@@ -341,6 +347,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [nb_segment]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = signal_channels
         self.header["spike_channels"] = spike_channels