Merge branch 'master' into fix_openephys_stream

Author: h-mayorquin

.github/workflows/io-test.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ['3.9', '3.12']
+        python-version: ['3.9', '3.13']
     defaults:
       # by default run in bash mode (required for conda usage)
       run:

environment_testing.yml

@@ -4,6 +4,3 @@ channels:
 dependencies:
   - datalad
   - pip
-  # temporary have this here for IO testing while we decide how to deal with
-  # external packages not 2.0 ready
-  - numpy=1.26.4

neo/io/klustakwikio.py

@@ -197,7 +197,7 @@ def _load_spike_times(self, fetfilename):
             names.append("spike_time")
 
         # Load into recarray
-        data = np.recfromtxt(fetfilename, names=names, skip_header=1, delimiter=" ")
+        data = np.genfromtxt(fetfilename, names=names, skip_header=1, delimiter=" ")
 
         # get features
         features = np.array([data[f"fet{n}"] for n in range(nbFeatures)])

neo/rawio/blackrockrawio.py

@@ -1365,7 +1365,7 @@ def __match_nsx_and_nev_segment_ids(self, nsx_nb):
 
                     # Show warning if spikes do not fit any segment (+- 1 sampling 'tick')
                     # Spike should belong to segment before
-                    mask_outside = (ev_ids == i) & (data["timestamp"] < int(seg["timestamp"]) - nsx_offset - nsx_period)
+                    mask_outside = (ev_ids == i) & (data["timestamp"] < int(seg["timestamp"]) - int(nsx_offset) - int(nsx_period))
 
                     if len(data[mask_outside]) > 0:
                         warnings.warn(f"Spikes outside any segment. Detected on segment #{i}")
@@ -1995,6 +1995,7 @@ def __get_nsx_param_variant_a(self, nsx_nb):
         else:
             units = "uV"
 
+        
         nsx_parameters = {
             "nb_data_points": int(
                 (self.__get_file_size(filename) - bytes_in_headers)
@@ -2003,8 +2004,8 @@ def __get_nsx_param_variant_a(self, nsx_nb):
             ),
             "labels": labels,
             "units": np.array([units] * self.__nsx_basic_header[nsx_nb]["channel_count"]),
-            "min_analog_val": -1 * np.array(dig_factor),
-            "max_analog_val": np.array(dig_factor),
+            "min_analog_val": -1 * np.array(dig_factor, dtype="float"),
+            "max_analog_val": np.array(dig_factor, dtype="float"),
             "min_digital_val": np.array([-1000] * self.__nsx_basic_header[nsx_nb]["channel_count"]),
             "max_digital_val": np.array([1000] * self.__nsx_basic_header[nsx_nb]["channel_count"]),
             "timestamp_resolution": 30000,

neo/rawio/micromedrawio.py

@@ -52,7 +52,6 @@ def __init__(self, filename=""):
 
     def _parse_header(self):
 
-        self._buffer_descriptions = {0: {0: {}}}
 
         with open(self.filename, "rb") as fid:
             f = StructFile(fid)
@@ -67,6 +66,7 @@ def _parse_header(self):
             rec_datetime = datetime.datetime(year + 1900, month, day, hour, minute, sec)
 
             Data_Start_Offset, Num_Chan, Multiplexer, Rate_Min, Bytes = f.read_f("IHHHH", offset=138)
+            sig_dtype = "u" + str(Bytes)
 
             # header version
             (header_version,) = f.read_f("b", offset=175)
@@ -99,25 +99,37 @@ def _parse_header(self):
                 if zname != zname2.decode("ascii").strip(" "):
                     raise NeoReadWriteError("expected the zone name to match")
 
-            # raw signals memmap
-            sig_dtype = "u" + str(Bytes)
-            signal_shape = get_memmap_shape(self.filename, sig_dtype, num_channels=Num_Chan, offset=Data_Start_Offset)
-            buffer_id = "0"
-            stream_id = "0"
-            self._buffer_descriptions[0][0][buffer_id] = {
-                "type": "raw",
-                "file_path": str(self.filename),
-                "dtype": sig_dtype,
-                "order": "C",
-                "file_offset": 0,
-                "shape": signal_shape,
-            }
+
+            # "TRONCA" zone define segments
+            zname2, pos, length = zones["TRONCA"]
+            f.seek(pos)
+            # this number avoid a infinite loop in case of corrupted  TRONCA zone (seg_start!=0 and trace_offset!=0)
+            max_segments = 100
+            self.info_segments = []
+            for i in range(max_segments):
+                # 4 bytes u4 each
+                seg_start = int(np.frombuffer(f.read(4), dtype="u4")[0])
+                trace_offset = int(np.frombuffer(f.read(4), dtype="u4")[0])
+                if seg_start == 0 and trace_offset == 0:
+                    break
+                else:
+                    self.info_segments.append((seg_start, trace_offset))
+            
+            if len(self.info_segments) == 0:
+                # one unique segment = general case
+                self.info_segments.append((0, 0))
+
+            nb_segment = len(self.info_segments)
 
             # Reading Code Info
             zname2, pos, length = zones["ORDER"]
             f.seek(pos)
             code = np.frombuffer(f.read(Num_Chan * 2), dtype="u2")
 
+            # unique stream and buffer
+            buffer_id = "0"
+            stream_id = "0"
+
             units_code = {-1: "nV", 0: "uV", 1: "mV", 2: 1, 100: "percent", 101: "dimensionless", 102: "dimensionless"}
             signal_channels = []
             sig_grounds = []
@@ -140,10 +152,8 @@ def _parse_header(self):
                 (sampling_rate,) = f.read_f("H")
                 sampling_rate *= Rate_Min
                 chan_id = str(c)
+                signal_channels.append((chan_name, chan_id, sampling_rate, sig_dtype, units, gain, offset, stream_id, buffer_id))
 
-                signal_channels.append(
-                    (chan_name, chan_id, sampling_rate, sig_dtype, units, gain, offset, stream_id, buffer_id)
-                )
 
             signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype)
 
@@ -155,6 +165,32 @@ def _parse_header(self):
                 raise NeoReadWriteError("The sampling rates must be the same across signal channels")
             self._sampling_rate = float(np.unique(signal_channels["sampling_rate"])[0])
 
+            # memmap traces buffer
+            full_signal_shape = get_memmap_shape(self.filename, sig_dtype, num_channels=Num_Chan, offset=Data_Start_Offset)
+            seg_limits = [trace_offset for seg_start, trace_offset in self.info_segments] + [full_signal_shape[0]]
+            self._t_starts = []
+            self._buffer_descriptions = {0 :{}}
+            for seg_index in range(nb_segment):
+                seg_start, trace_offset = self.info_segments[seg_index]
+                self._t_starts.append(seg_start / self._sampling_rate)
+
+                start = seg_limits[seg_index]
+                stop = seg_limits[seg_index + 1]
+                
+                shape = (stop - start, Num_Chan)
+                file_offset = Data_Start_Offset + ( start * np.dtype(sig_dtype).itemsize * Num_Chan)
+                self._buffer_descriptions[0][seg_index] = {}
+                self._buffer_descriptions[0][seg_index][buffer_id] = {
+                    "type" : "raw",
+                    "file_path" : str(self.filename),
+                    "dtype" : sig_dtype,
+                    "order": "C",
+                    "file_offset" : file_offset,
+                    "shape" : shape,
+                }
+
+
+
             # Event channels
             event_channels = []
             event_channels.append(("Trigger", "", "event"))
@@ -176,13 +212,18 @@ def _parse_header(self):
                 dtype = np.dtype(ev_dtype)
                 rawevent = np.memmap(self.filename, dtype=dtype, mode="r", offset=pos, shape=length // dtype.itemsize)
 
-                keep = (
-                    (rawevent["start"] >= rawevent["start"][0])
-                    & (rawevent["start"] < signal_shape[0])
-                    & (rawevent["start"] != 0)
-                )
-                rawevent = rawevent[keep]
-                self._raw_events.append(rawevent)
+                # important : all events timing are related to the first segment t_start
+                self._raw_events.append([])
+                for seg_index in range(nb_segment):
+                    left_lim = seg_limits[seg_index]
+                    right_lim = seg_limits[seg_index + 1]
+                    keep = (
+                        (rawevent["start"] >= left_lim)
+                        & (rawevent["start"] < right_lim)
+                        & (rawevent["start"] != 0)
+                    )
+                    self._raw_events[-1].append(rawevent[keep])
+
 
             # No spikes
             spike_channels = []
@@ -191,7 +232,7 @@ def _parse_header(self):
             # fille into header dict
             self.header = {}
             self.header["nb_block"] = 1
-            self.header["nb_segment"] = [1]
+            self.header["nb_segment"] = [nb_segment]
             self.header["signal_buffers"] = signal_buffers
             self.header["signal_streams"] = signal_streams
             self.header["signal_channels"] = signal_channels
@@ -216,38 +257,40 @@ def _source_name(self):
         return self.filename
 
     def _segment_t_start(self, block_index, seg_index):
-        return 0.0
+        return self._t_starts[seg_index]
 
     def _segment_t_stop(self, block_index, seg_index):
-        sig_size = self.get_signal_size(block_index, seg_index, 0)
-        t_stop = sig_size / self._sampling_rate
-        return t_stop
+        duration = self.get_signal_size(block_index, seg_index, stream_index=0) / self._sampling_rate
+        return duration + self.segment_t_start(block_index, seg_index)
 
     def _get_signal_t_start(self, block_index, seg_index, stream_index):
-        if stream_index != 0:
-            raise ValueError("`stream_index` must be 0")
-        return 0.0
+        assert stream_index == 0
+        return self._t_starts[seg_index]
 
     def _spike_count(self, block_index, seg_index, unit_index):
         return 0
 
     def _event_count(self, block_index, seg_index, event_channel_index):
-        n = self._raw_events[event_channel_index].size
+        n = self._raw_events[event_channel_index][seg_index].size
         return n
 
     def _get_event_timestamps(self, block_index, seg_index, event_channel_index, t_start, t_stop):
 
-        raw_event = self._raw_events[event_channel_index]
+        raw_event = self._raw_events[event_channel_index][seg_index]
+
+        # important : all events timing are related to the first segment t_start
+        seg_start0, _ = self.info_segments[0]
 
         if t_start is not None:
-            keep = raw_event["start"] >= int(t_start * self._sampling_rate)
+            keep = raw_event["start"] + seg_start0 >= int(t_start * self._sampling_rate)
             raw_event = raw_event[keep]
 
         if t_stop is not None:
-            keep = raw_event["start"] <= int(t_stop * self._sampling_rate)
+            keep = raw_event["start"] + seg_start0 <= int(t_stop * self._sampling_rate)
             raw_event = raw_event[keep]
 
-        timestamp = raw_event["start"]
+        timestamp = raw_event["start"] + seg_start0
+
         if event_channel_index < 2:
             durations = None
         else:

neo/test/iotest/test_asciisignalio.py

@@ -166,10 +166,10 @@ def test_skiprows(self):
         self.assertEqual(signal.units, pq.V)
         assert_array_equal(signal.times, [0.0, 1.0, 2.0, 3.0] * pq.s)
         assert_array_equal(signal.times.magnitude, [0.0, 1.0, 2.0, 3.0])
-        assert_array_equal(signal[0].magnitude, -64.8)
-        assert_array_equal(signal[1].magnitude, -64.6)
-        assert_array_equal(signal[2].magnitude, -64.3)
-        assert_array_equal(signal[3].magnitude, -66)
+        assert_array_almost_equal(signal[0].magnitude, -64.8, decimal=5)
+        assert_array_almost_equal(signal[1].magnitude, -64.6, decimal=5)
+        assert_array_almost_equal(signal[2].magnitude, -64.3, decimal=5)
+        assert_array_almost_equal(signal[3].magnitude, -66, decimal=5)
         assert_array_almost_equal(np.asarray(signal).flatten(), np.array([-64.8, -64.6, -64.3, -66]), decimal=5)
 
         os.remove(filename)
@@ -195,11 +195,11 @@ def test_usecols(self):
         self.assertEqual(signal.units, pq.V)
         assert_array_equal(signal.times, [0.0, 1.0, 2.0, 3.0, 4.0] * pq.s)
         assert_array_equal(signal.times.magnitude, [0.0, 1.0, 2.0, 3.0, 4.0])
-        assert_array_equal(signal[0].magnitude, 0.5)
-        assert_array_equal(signal[1].magnitude, 0.6)
-        assert_array_equal(signal[2].magnitude, 0.7)
-        assert_array_equal(signal[3].magnitude, 0.8)
-        assert_array_equal(signal[4].magnitude, 1.4)
+        assert_array_almost_equal(signal[0].magnitude, 0.5, decimal=5)
+        assert_array_almost_equal(signal[1].magnitude, 0.6, decimal=5)
+        assert_array_almost_equal(signal[2].magnitude, 0.7, decimal=5)
+        assert_array_almost_equal(signal[3].magnitude, 0.8, decimal=5)
+        assert_array_almost_equal(signal[4].magnitude, 1.4, decimal=5)
         assert_array_almost_equal(np.asarray(signal).flatten(), np.array([0.5, 0.6, 0.7, 0.8, 1.4]), decimal=5)
 
         os.remove(filename)

neo/test/iotest/test_axographio.py

@@ -9,7 +9,7 @@
 from neo.test.iotest.common_io_test import BaseTestIO
 
 import numpy as np
-from numpy.testing import assert_equal
+from numpy.testing import assert_equal, assert_almost_equal
 import quantities as pq
 from neo.test.rawiotest.test_axographrawio import TestAxographRawIO
 
@@ -35,9 +35,9 @@ def test_version_1(self):
         target = np.array([[-5.5078130], [-3.1171880], [+1.6640626], [+1.6640626], [+4.0546880]], dtype=np.float32)
         assert_equal(arr, target)
 
-        assert_equal(sig.t_start, 0.0005000000237487257 * pq.s)
+        assert_almost_equal(sig.t_start, 0.0005000000237487257 * pq.s, decimal=9)
 
-        assert_equal(sig.sampling_period, 0.0005000010132789612 * pq.s)
+        assert_almost_equal(sig.sampling_period, 0.0005000010132789612 * pq.s, decimal=9)
 
     def test_version_2(self):
         """Test reading a version 2 AxoGraph file"""
@@ -87,9 +87,9 @@ def test_version_2(self):
         target = np.array([[0.3125], [9.6875], [9.6875], [9.6875], [9.3750]], dtype=np.float32)
         assert_equal(arr, target)
 
-        assert_equal(sig.t_start, 0.00009999999747378752 * pq.s)
+        assert_almost_equal(sig.t_start, 0.00009999999747378752 * pq.s, decimal=9)
 
-        assert_equal(sig.sampling_period, 0.00009999999747378750 * pq.s)
+        assert_almost_equal(sig.sampling_period, 0.00009999999747378750 * pq.s, decimal=9)
 
     def test_version_5(self):
         """Test reading a version 5 AxoGraph file"""
@@ -169,7 +169,7 @@ def test_file_written_by_axographio_package_without_linearsequence(self):
 
         assert_equal(sig.t_start, 0 * pq.s)
 
-        assert_equal(sig.sampling_period, 0.009999999999999787 * pq.s)
+        assert_almost_equal(sig.sampling_period, 0.009999999999999787 * pq.s, decimal=9)
 
     def test_file_with_corrupt_comment(self):
         """Test reading a file with a corrupt comment"""

neo/test/rawiotest/test_micromedrawio.py

@@ -8,14 +8,50 @@
 
 from neo.test.rawiotest.common_rawio_test import BaseTestRawIO
 
+import numpy as np
 
 class TestMicromedRawIO(
     BaseTestRawIO,
     unittest.TestCase,
 ):
     rawioclass = MicromedRawIO
     entities_to_download = ["micromed"]
-    entities_to_test = ["micromed/File_micromed_1.TRC"]
+    entities_to_test = [
+        "micromed/File_micromed_1.TRC",
+        "micromed/File_mircomed2.TRC",
+        "micromed/File_mircomed2_2segments.TRC",
+    ]
+
+    def test_micromed_multi_segments(self):
+        file_full = self.get_local_path("micromed/File_mircomed2.TRC")
+        file_splitted = self.get_local_path("micromed/File_mircomed2_2segments.TRC")
+        
+        # the second file contains 2 pieces of the first file
+        # so it is 2 segments with the same traces but reduced
+        # note that traces in the splited can differ at the very end of the cut
+
+        reader1 = MicromedRawIO(file_full)
+        reader1.parse_header()
+        assert reader1.segment_count(block_index=0) == 1
+        assert reader1.get_signal_t_start(block_index=0, seg_index=0, stream_index=0) == 0.
+        traces1 = reader1.get_analogsignal_chunk(stream_index=0)
+
+        reader2 = MicromedRawIO(file_splitted)
+        reader2.parse_header()
+        print(reader2)
+        assert reader2.segment_count(block_index=0) == 2
+
+        # check that pieces of the second file is equal to the first file (except a truncation at the end)
+        for seg_index in range(2):
+            t_start = reader2.get_signal_t_start(block_index=0, seg_index=seg_index, stream_index=0)
+            assert t_start > 0
+            sr = reader2.get_signal_sampling_rate(stream_index=0)
+            ind_start = int(t_start * sr)
+            traces2 = reader2.get_analogsignal_chunk(block_index=0, seg_index=seg_index, stream_index=0)
+            traces1_chunk = traces1[ind_start: ind_start+traces2.shape[0]]
+            # we remove the last 100 sample because tools that cut traces is truncating the last buffer
+            assert np.array_equal(traces2[:-100], traces1_chunk[:-100])
+
 
 
 if __name__ == "__main__":

pyproject.toml

@@ -81,7 +81,7 @@ docs = [
     "nixio",
     "pynwb",
     "igor2",
-    "numpy<2.0" # https://github.com/NeuralEnsemble/python-neo/pull/1612
+    "numpy>=2.0"
 ]
 
 dev = [