Merge branch 'master' into remove-copy

Author: zm711

.github/workflows/core-test.yml

@@ -26,24 +26,9 @@ jobs:
       matrix:
         os: ["ubuntu-latest", "windows-latest"]
         # "macos-latest",
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
-        numpy-version: ['1.20.3', '1.21.6', '1.22.4', '1.23.5', '1.24.1', '1.25.1', '1.26.4']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+        numpy-version: ['1.22.4', '1.23.5', '1.24.1', '1.25.1', '1.26.4']
         exclude:
-           - python-version: '3.8'
-             numpy-version: '1.25.1'
-           - python-version: '3.8'
-             numpy-version: '1.26.4'
-           - python-version: '3.10'
-             numpy-version: '1.20.3'
-           - python-version: '3.11'
-             numpy-version: '1.20.3'
-           - python-version: '3.11'
-             numpy-version: '1.21.6'
-           - python-version: '3.12'
-             numpy-version: '1.20.3'
-           # python 3.12 only works on latest numpy
-           - python-version: '3.12'
-             numpy-version: '1.21.6'
            - python-version: '3.12'
              numpy-version: '1.22.4'
            - python-version: '3.12'

README.rst

@@ -64,7 +64,8 @@ grant agreement FETPI-015879 (FACETS), by the European Union Seventh Framework P
 under grant agreements no. 269921 (BrainScaleS) and no. 604102 (HBP),
 and by the European Union’s Horizon 2020 Framework Programme for
 Research and Innovation under the Specific Grant Agreements No. 720270 (Human Brain Project SGA1),
-No. 785907 (Human Brain Project SGA2) and No. 945539 (Human Brain Project SGA3).
+No. 785907 (Human Brain Project SGA2) and No. 945539 (Human Brain Project SGA3),
+and by the European Union's Research and Innovation Program Horizon Europe Grant Agreement No. 101147319 (EBRAINS 2.0).
 
 .. _OpenElectrophy: https://github.com/OpenElectrophy/OpenElectrophy
 .. _Elephant: http://neuralensemble.org/elephant

neo/io/igorproio.py

@@ -133,7 +133,24 @@ def _wave_to_analogsignal(self, content, dirpath):
         header = content["wave_header"]
         name = str(header["bname"].decode("utf-8"))
         units = "".join([x.decode() for x in header["dataUnits"]])
-        time_units = "".join([x.decode() for x in header["xUnits"]])
+        if "xUnits" in header:
+            # "xUnits" is used in Versions 1, 2, 3 of .pxp files
+            time_units = "".join([x.decode() for x in header["xUnits"]])
+        elif "dimUnits" in header:
+            # Version 5 uses "dimUnits"
+            # see https://github.com/AFM-analysis/igor2/blob/43fccf51714661fb96372e8119c59e17ce01f683/igor2/binarywave.py#L501
+            _time_unit_structure = header["dimUnits"].ravel()
+            # For the files we've seen so far, the first element of _time_unit_structure contains the units.
+            # If someone has a file for which this assumption does not hold an Exception will be raised.
+            if not all([element == b"" for element in _time_unit_structure[1:]]):
+                raise Exception(
+                    "Neo cannot yet handle the units in this file. "
+                    "Please create a new issue in the Neo issue tracker at "
+                    "https://github.com/NeuralEnsemble/python-neo/issues/new/choose"
+                )
+            time_units = _time_unit_structure[0].decode()
+        else:
+            time_units = ""
         if len(time_units) == 0:
             time_units = "s"
         try:

neo/io/neuronexusio.py

@@ -8,4 +8,4 @@ class NeuroNexusIO(NeuroNexusRawIO, BaseFromRaw):
 
     def __init__(self, filename):
         NeuroNexusRawIO.__init__(self, filename=filename)
-        BaseFromRaw.__init__(self, filename)
+        BaseFromRaw.__init__(self, filename)

neo/rawio/alphaomegarawio.py

@@ -35,6 +35,7 @@
 
 from .baserawio import (
     BaseRawIO,
+    _signal_buffer_dtype,
     _signal_stream_dtype,
     _signal_channel_dtype,
     _spike_channel_dtype,
@@ -652,8 +653,9 @@ def _parse_header(self):
         # could be loaded in any order
         self._merge_segments()
 
+        buffer_id = ""
         signal_streams = set(
-            (stream_name, stream_id)
+            (stream_name, stream_id, buffer_id)
             for segment in self._segments
             for stream in segment["streams"]
             for stream_name, _, stream_id in self.STREAM_CHANNELS
@@ -662,7 +664,9 @@ def _parse_header(self):
         signal_streams = list(signal_streams)
         signal_streams.sort(key=lambda x: x[1])
         signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)
+        signal_buffers = np.array([], dtype=_signal_buffer_dtype)
 
+        buffer_id = ""
         signal_channels = set(
             (
                 channel["name"],
@@ -672,11 +676,12 @@ def _parse_header(self):
                 "uV",
                 channel["gain"] / channel["bit_resolution"],
                 0,
-                stream,
+                stream_id,
+                buffer_id,
             )
             for segment in self._segments
-            for stream in segment["streams"]
-            for channel_id, channel in segment["streams"][stream].items()
+            for stream_id in segment["streams"]
+            for channel_id, channel in segment["streams"][stream_id].items()
         )
         signal_channels = list(signal_channels)
         signal_channels.sort(key=lambda x: (x[7], x[0]))
@@ -709,6 +714,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [len(self._segments)]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = signal_channels
         self.header["spike_channels"] = spike_channels

neo/rawio/axographrawio.py

@@ -158,6 +158,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -811,7 +812,19 @@ def _scan_axograph_file(self):
                     self.logger.debug("initial data: {array[:5] * gain + offset}")
 
                     # channel_info will be cast to _signal_channel_dtype
-                    channel_info = (name, str(i), 1 / sampling_period, f.byte_order + dtype, units, gain, offset, "0")
+                    buffer_id = ""
+                    stream_id = "0"
+                    channel_info = (
+                        name,
+                        str(i),
+                        1 / sampling_period,
+                        f.byte_order + dtype,
+                        units,
+                        gain,
+                        offset,
+                        stream_id,
+                        buffer_id,
+                    )
 
                     self.logger.debug("channel_info: {channel_info}")
                     self.logger.debug("")
@@ -1230,13 +1243,15 @@ def _scan_axograph_file(self):
         event_channels.append(("AxoGraph Intervals", "", "epoch"))
 
         if len(sig_channels) > 0:
-            signal_streams = [("Signals", "0")]
+            signal_streams = [("Signals", "0", "")]
         else:
             signal_streams = []
+        signal_buffers = []
 
         # organize header
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [1]
+        self.header["signal_buffers"] = np.array(signal_buffers, dtype=_signal_buffer_dtype)
         self.header["signal_streams"] = np.array(signal_streams, dtype=_signal_stream_dtype)
         self.header["signal_channels"] = np.array(sig_channels, dtype=_signal_channel_dtype)
         self.header["event_channels"] = np.array(event_channels, dtype=_event_channel_dtype)

neo/rawio/axonarawio.py

@@ -25,6 +25,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -199,6 +200,7 @@ def _parse_header(self):
         params["set"]["sampling_rate"] = int(set_dict["rawRate"])
 
         # SCAN BIN FILE
+        signal_buffers = []
         signal_streams = []
         signal_channels = []
         if self.bin_file:
@@ -289,6 +291,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [1]
+        self.header["signal_buffers"] = np.array(signal_buffers, dtype=_signal_buffer_dtype)
         self.header["signal_streams"] = np.array(signal_streams, dtype=_signal_stream_dtype)
         self.header["signal_channels"] = np.array(signal_channels, dtype=_signal_channel_dtype)
         self.header["spike_channels"] = np.array(spike_channels, dtype=_spike_channel_dtype)
@@ -315,7 +318,7 @@ def _parse_header(self):
 
     def _get_signal_streams_header(self):
         # create signals stream information (we always expect a single stream)
-        return np.array([("stream 0", "0")], dtype=_signal_stream_dtype)
+        return np.array([("stream 0", "0", "")], dtype=_signal_stream_dtype)
 
     def _segment_t_start(self, block_index, seg_index):
         return 0.0
@@ -638,10 +641,11 @@ def _get_signal_chan_header(self):
                 chan_id = str(cntr)
                 gain = gain_list[cntr]
                 stream_id = "0"
+                buffer_id = ""
                 # the sampling rate information is stored in the set header
                 # and not in the bin file
                 sr = self.file_parameters["set"]["sampling_rate"]
-                sig_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id))
+                sig_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id, buffer_id))
 
         return np.array(sig_channels, dtype=_signal_channel_dtype)
 

neo/rawio/axonrawio.py

@@ -56,6 +56,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -227,12 +228,16 @@ def _parse_header(self):
             else:
                 gain, offset = 1.0, 0.0
             stream_id = "0"
-            signal_channels.append((name, str(chan_id), self._sampling_rate, sig_dtype, units, gain, offset, stream_id))
+            buffer_id = "0"
+            signal_channels.append(
+                (name, str(chan_id), self._sampling_rate, sig_dtype, units, gain, offset, stream_id, buffer_id)
+            )
 
         signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype)
 
-        # one unique signal stream
-        signal_streams = np.array([("Signals", "0")], dtype=_signal_stream_dtype)
+        # one unique signal stream and buffer
+        signal_buffers = np.array([("Signals", "0")], dtype=_signal_buffer_dtype)
+        signal_streams = np.array([("Signals", "0", "0")], dtype=_signal_stream_dtype)
 
         # only one events channel : tag
         # In ABF timstamps are not attached too any particular segment
@@ -258,6 +263,7 @@ def _parse_header(self):
         self.header = {}
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [nb_segment]
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
         self.header["signal_channels"] = signal_channels
         self.header["spike_channels"] = spike_channels

neo/rawio/baserawio.py

@@ -86,9 +86,18 @@
 
 error_header = "Header is not read yet, do parse_header() first"
 
+_signal_buffer_dtype = [
+    ("name", "U64"),  # not necessarily unique
+    ("id", "U64"),  # must be unique
+]
+# To be left an empty array if the concept of buffer is undefined for a reader.
 _signal_stream_dtype = [
     ("name", "U64"),  # not necessarily unique
     ("id", "U64"),  # must be unique
+    (
+        "buffer_id",
+        "U64",
+    ),  # should be "" (empty string) when the stream is not nested under a buffer or the buffer is undefined for some reason.
 ]
 
 _signal_channel_dtype = [
@@ -100,6 +109,7 @@
     ("gain", "float64"),
     ("offset", "float64"),
     ("stream_id", "U64"),
+    ("buffer_id", "U64"),
 ]
 
 # TODO for later: add t_start and length in _signal_channel_dtype

neo/rawio/bci2000rawio.py

@@ -15,6 +15,7 @@
     BaseRawIO,
     _signal_channel_dtype,
     _signal_stream_dtype,
+    _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
 )
@@ -49,8 +50,10 @@ def _parse_header(self):
         self.header["nb_block"] = 1
         self.header["nb_segment"] = [1]
 
-        # one unique stream
-        signal_streams = np.array([("Signals", "0")], dtype=_signal_stream_dtype)
+        # one unique stream and buffer
+        signal_buffers = np.array(("Signals", "0"), dtype=_signal_buffer_dtype)
+        signal_streams = np.array([("Signals", "0", "0")], dtype=_signal_stream_dtype)
+        self.header["signal_buffers"] = signal_buffers
         self.header["signal_streams"] = signal_streams
 
         sig_channels = []
@@ -78,7 +81,8 @@ def _parse_header(self):
                 offset = float(offset)
 
             stream_id = "0"
-            sig_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id))
+            buffer_id = "0"
+            sig_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id, buffer_id))
         self.header["signal_channels"] = np.array(sig_channels, dtype=_signal_channel_dtype)
 
         self.header["spike_channels"] = np.array([], dtype=_spike_channel_dtype)