Merge branch 'master' into update-copyright

Author: samuelgarcia

.github/workflows/io-test.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ['3.9', '3.12']
+        python-version: ['3.9', '3.13']
     defaults:
       # by default run in bash mode (required for conda usage)
       run:

environment_testing.yml

@@ -4,6 +4,3 @@ channels:
 dependencies:
   - datalad
   - pip
-  # temporary have this here for IO testing while we decide how to deal with
-  # external packages not 2.0 ready
-  - numpy=1.26.4

neo/core/analogsignal.py

@@ -200,7 +200,10 @@ def __new__(
         """
         if copy is not None:
             raise ValueError(
-                "`copy` is now deprecated in Neo due to removal in NumPy 2.0 and will be removed in 0.15.0."
+                "`copy` is now deprecated in Neo due to removal in Quantites to support Numpy 2.0. "
+                "In order to facilitate the deprecation copy can be set to None but will raise an "
+                "error if set to True/False since this will silently do nothing. This argument will be completely "
+                "removed in Neo 0.15.0. Please update your code base as necessary."
             )
 
         signal = cls._rescale(signal, units=units)
@@ -210,7 +213,7 @@ def __new__(
             obj.shape = (-1, 1)
 
         if t_start is None:
-            raise ValueError("t_start cannot be None")
+            raise ValueError("`t_start` cannot be None")
         obj._t_start = t_start
 
         obj._sampling_rate = _get_sampling_rate(sampling_rate, sampling_period)

neo/core/imagesequence.py

@@ -124,7 +124,10 @@ def __new__(
 
         if copy is not None:
             raise ValueError(
-                "`copy` is now deprecated in Neo due to removal in NumPy 2.0 and will be removed in 0.15.0."
+                "`copy` is now deprecated in Neo due to removal in Quantites to support Numpy 2.0. "
+                "In order to facilitate the deprecation copy can be set to None but will raise an "
+                "error if set to True/False since this will silently do nothing. This argument will be completely "
+                "removed in Neo 0.15.0. Please update your code base as necessary."
             )
 
         if spatial_scale is None:

neo/core/irregularlysampledsignal.py

@@ -174,7 +174,10 @@ def __new__(
 
         if copy is not None:
             raise ValueError(
-                "`copy` is now deprecated in Neo due to removal in NumPy 2.0 and will be removed in 0.15.0."
+                "`copy` is now deprecated in Neo due to removal in Quantites to support Numpy 2.0. "
+                "In order to facilitate the deprecation copy can be set to None but will raise an "
+                "error if set to True/False since this will silently do nothing. This argument will be completely "
+                "removed in Neo 0.15.0. Please update your code base as necessary."
             )
 
         signal = cls._rescale(signal, units=units)

neo/core/spiketrain.py

@@ -194,7 +194,12 @@ def normalize_times_array(times, units=None, dtype=None, copy=None):
     """
 
     if copy is not None:
-        raise ValueError("`copy` is now deprecated in Neo due to removal in NumPy 2.0 and will be removed in 0.15.0.")
+        raise ValueError(
+            "`copy` is now deprecated in Neo due to removal in Quantites to support Numpy 2.0. "
+            "In order to facilitate the deprecation copy can be set to None but will raise an "
+            "error if set to True/False since this will silently do nothing. This argument will be completely "
+            "removed in Neo 0.15.0. Please update your code base as necessary."
+            )
 
     if dtype is None:
         if not hasattr(times, "dtype"):
@@ -352,7 +357,10 @@ def __new__(
         """
         if copy is not None:
             raise ValueError(
-                "`copy` is now deprecated in Neo due to removal in NumPy 2.0 and will be removed in 0.15.0."
+                "`copy` is now deprecated in Neo due to removal in Quantites to support Numpy 2.0. "
+                "In order to facilitate the deprecation copy can be set to None but will raise an "
+                "error if set to True/False since this will silently do nothing. This argument will be completely "
+                "removed in Neo 0.15.0. Please update your code base as necessary."
             )
 
         if len(times) != 0 and waveforms is not None and len(times) != waveforms.shape[0]:

neo/io/klustakwikio.py

@@ -197,7 +197,7 @@ def _load_spike_times(self, fetfilename):
             names.append("spike_time")
 
         # Load into recarray
-        data = np.recfromtxt(fetfilename, names=names, skip_header=1, delimiter=" ")
+        data = np.genfromtxt(fetfilename, names=names, skip_header=1, delimiter=" ")
 
         # get features
         features = np.array([data[f"fet{n}"] for n in range(nbFeatures)])

neo/rawio/blackrockrawio.py

@@ -473,6 +473,7 @@ def _parse_header(self):
                         segment_mask = ev_ids == data_bl
                         if data[segment_mask].size > 0:
                             t = data[segment_mask][-1]["timestamp"] / self.__nev_basic_header["timestamp_resolution"]
+
                             max_nev_time = max(max_nev_time, t)
                     if max_nev_time > t_stop:
                         t_stop = max_nev_time
@@ -680,7 +681,8 @@ def _get_timestamp_slice(self, timestamp, seg_index, t_start, t_stop):
             if t_start is None:
                 t_start = self._seg_t_starts[seg_index]
             if t_stop is None:
-                t_stop = self._seg_t_stops[seg_index]
+                t_stop = self._seg_t_stops[seg_index] + 1 / float(
+                    self.__nev_basic_header['timestamp_resolution'])
 
         if t_start is None:
             ind_start = None
@@ -713,10 +715,16 @@ def _get_spike_raw_waveforms(self, block_index, seg_index, unit_index, t_start,
         )
         unit_spikes = all_spikes[mask]
 
-        wf_dtype = self.__nev_params("waveform_dtypes")[channel_id]
-        wf_size = self.__nev_params("waveform_size")[channel_id]
+        wf_dtype = self.__nev_params('waveform_dtypes')[channel_id]
+        wf_size = self.__nev_params('waveform_size')[channel_id]
+        wf_byte_size = np.dtype(wf_dtype).itemsize * wf_size
+
+        dt1 = [
+            ('extra', 'S{}'.format(unit_spikes['waveform'].dtype.itemsize - wf_byte_size)),
+            ('ch_waveform', 'S{}'.format(wf_byte_size))]
+
+        waveforms = unit_spikes['waveform'].view(dt1)['ch_waveform'].flatten().view(wf_dtype)
 
-        waveforms = unit_spikes["waveform"].flatten().view(wf_dtype)
         waveforms = waveforms.reshape(int(unit_spikes.size), 1, int(wf_size))
 
         timestamp = unit_spikes["timestamp"]
@@ -1357,7 +1365,7 @@ def __match_nsx_and_nev_segment_ids(self, nsx_nb):
 
                     # Show warning if spikes do not fit any segment (+- 1 sampling 'tick')
                     # Spike should belong to segment before
-                    mask_outside = (ev_ids == i) & (data["timestamp"] < int(seg["timestamp"]) - nsx_offset - nsx_period)
+                    mask_outside = (ev_ids == i) & (data["timestamp"] < int(seg["timestamp"]) - int(nsx_offset) - int(nsx_period))
 
                     if len(data[mask_outside]) > 0:
                         warnings.warn(f"Spikes outside any segment. Detected on segment #{i}")
@@ -1987,6 +1995,7 @@ def __get_nsx_param_variant_a(self, nsx_nb):
         else:
             units = "uV"
 
+        
         nsx_parameters = {
             "nb_data_points": int(
                 (self.__get_file_size(filename) - bytes_in_headers)
@@ -1995,8 +2004,8 @@ def __get_nsx_param_variant_a(self, nsx_nb):
             ),
             "labels": labels,
             "units": np.array([units] * self.__nsx_basic_header[nsx_nb]["channel_count"]),
-            "min_analog_val": -1 * np.array(dig_factor),
-            "max_analog_val": np.array(dig_factor),
+            "min_analog_val": -1 * np.array(dig_factor, dtype="float"),
+            "max_analog_val": np.array(dig_factor, dtype="float"),
             "min_digital_val": np.array([-1000] * self.__nsx_basic_header[nsx_nb]["channel_count"]),
             "max_digital_val": np.array([1000] * self.__nsx_basic_header[nsx_nb]["channel_count"]),
             "timestamp_resolution": 30000,

neo/rawio/micromedrawio.py

@@ -52,7 +52,6 @@ def __init__(self, filename=""):
 
     def _parse_header(self):
 
-        self._buffer_descriptions = {0: {0: {}}}
 
         with open(self.filename, "rb") as fid:
             f = StructFile(fid)
@@ -67,6 +66,7 @@ def _parse_header(self):
             rec_datetime = datetime.datetime(year + 1900, month, day, hour, minute, sec)
 
             Data_Start_Offset, Num_Chan, Multiplexer, Rate_Min, Bytes = f.read_f("IHHHH", offset=138)
+            sig_dtype = "u" + str(Bytes)
 
             # header version
             (header_version,) = f.read_f("b", offset=175)
@@ -99,25 +99,37 @@ def _parse_header(self):
                 if zname != zname2.decode("ascii").strip(" "):
                     raise NeoReadWriteError("expected the zone name to match")
 
-            # raw signals memmap
-            sig_dtype = "u" + str(Bytes)
-            signal_shape = get_memmap_shape(self.filename, sig_dtype, num_channels=Num_Chan, offset=Data_Start_Offset)
-            buffer_id = "0"
-            stream_id = "0"
-            self._buffer_descriptions[0][0][buffer_id] = {
-                "type": "raw",
-                "file_path": str(self.filename),
-                "dtype": sig_dtype,
-                "order": "C",
-                "file_offset": 0,
-                "shape": signal_shape,
-            }
+
+            # "TRONCA" zone define segments
+            zname2, pos, length = zones["TRONCA"]
+            f.seek(pos)
+            # this number avoid a infinite loop in case of corrupted  TRONCA zone (seg_start!=0 and trace_offset!=0)
+            max_segments = 100
+            self.info_segments = []
+            for i in range(max_segments):
+                # 4 bytes u4 each
+                seg_start = int(np.frombuffer(f.read(4), dtype="u4")[0])
+                trace_offset = int(np.frombuffer(f.read(4), dtype="u4")[0])
+                if seg_start == 0 and trace_offset == 0:
+                    break
+                else:
+                    self.info_segments.append((seg_start, trace_offset))
+            
+            if len(self.info_segments) == 0:
+                # one unique segment = general case
+                self.info_segments.append((0, 0))
+
+            nb_segment = len(self.info_segments)
 
             # Reading Code Info
             zname2, pos, length = zones["ORDER"]
             f.seek(pos)
             code = np.frombuffer(f.read(Num_Chan * 2), dtype="u2")
 
+            # unique stream and buffer
+            buffer_id = "0"
+            stream_id = "0"
+
             units_code = {-1: "nV", 0: "uV", 1: "mV", 2: 1, 100: "percent", 101: "dimensionless", 102: "dimensionless"}
             signal_channels = []
             sig_grounds = []
@@ -140,10 +152,8 @@ def _parse_header(self):
                 (sampling_rate,) = f.read_f("H")
                 sampling_rate *= Rate_Min
                 chan_id = str(c)
+                signal_channels.append((chan_name, chan_id, sampling_rate, sig_dtype, units, gain, offset, stream_id, buffer_id))
 
-                signal_channels.append(
-                    (chan_name, chan_id, sampling_rate, sig_dtype, units, gain, offset, stream_id, buffer_id)
-                )
 
             signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype)
 
@@ -155,6 +165,32 @@ def _parse_header(self):
                 raise NeoReadWriteError("The sampling rates must be the same across signal channels")
             self._sampling_rate = float(np.unique(signal_channels["sampling_rate"])[0])
 
+            # memmap traces buffer
+            full_signal_shape = get_memmap_shape(self.filename, sig_dtype, num_channels=Num_Chan, offset=Data_Start_Offset)
+            seg_limits = [trace_offset for seg_start, trace_offset in self.info_segments] + [full_signal_shape[0]]
+            self._t_starts = []
+            self._buffer_descriptions = {0 :{}}
+            for seg_index in range(nb_segment):
+                seg_start, trace_offset = self.info_segments[seg_index]
+                self._t_starts.append(seg_start / self._sampling_rate)
+
+                start = seg_limits[seg_index]
+                stop = seg_limits[seg_index + 1]
+                
+                shape = (stop - start, Num_Chan)
+                file_offset = Data_Start_Offset + ( start * np.dtype(sig_dtype).itemsize * Num_Chan)
+                self._buffer_descriptions[0][seg_index] = {}
+                self._buffer_descriptions[0][seg_index][buffer_id] = {
+                    "type" : "raw",
+                    "file_path" : str(self.filename),
+                    "dtype" : sig_dtype,
+                    "order": "C",
+                    "file_offset" : file_offset,
+                    "shape" : shape,
+                }
+
+
+
             # Event channels
             event_channels = []
             event_channels.append(("Trigger", "", "event"))
@@ -176,13 +212,18 @@ def _parse_header(self):
                 dtype = np.dtype(ev_dtype)
                 rawevent = np.memmap(self.filename, dtype=dtype, mode="r", offset=pos, shape=length // dtype.itemsize)
 
-                keep = (
-                    (rawevent["start"] >= rawevent["start"][0])
-                    & (rawevent["start"] < signal_shape[0])
-                    & (rawevent["start"] != 0)
-                )
-                rawevent = rawevent[keep]
-                self._raw_events.append(rawevent)
+                # important : all events timing are related to the first segment t_start
+                self._raw_events.append([])
+                for seg_index in range(nb_segment):
+                    left_lim = seg_limits[seg_index]
+                    right_lim = seg_limits[seg_index + 1]
+                    keep = (
+                        (rawevent["start"] >= left_lim)
+                        & (rawevent["start"] < right_lim)
+                        & (rawevent["start"] != 0)
+                    )
+                    self._raw_events[-1].append(rawevent[keep])
+
 
             # No spikes
             spike_channels = []
@@ -191,7 +232,7 @@ def _parse_header(self):
             # fille into header dict
             self.header = {}
             self.header["nb_block"] = 1
-            self.header["nb_segment"] = [1]
+            self.header["nb_segment"] = [nb_segment]
             self.header["signal_buffers"] = signal_buffers
             self.header["signal_streams"] = signal_streams
             self.header["signal_channels"] = signal_channels
@@ -216,38 +257,40 @@ def _source_name(self):
         return self.filename
 
     def _segment_t_start(self, block_index, seg_index):
-        return 0.0
+        return self._t_starts[seg_index]
 
     def _segment_t_stop(self, block_index, seg_index):
-        sig_size = self.get_signal_size(block_index, seg_index, 0)
-        t_stop = sig_size / self._sampling_rate
-        return t_stop
+        duration = self.get_signal_size(block_index, seg_index, stream_index=0) / self._sampling_rate
+        return duration + self.segment_t_start(block_index, seg_index)
 
     def _get_signal_t_start(self, block_index, seg_index, stream_index):
-        if stream_index != 0:
-            raise ValueError("`stream_index` must be 0")
-        return 0.0
+        assert stream_index == 0
+        return self._t_starts[seg_index]
 
     def _spike_count(self, block_index, seg_index, unit_index):
         return 0
 
     def _event_count(self, block_index, seg_index, event_channel_index):
-        n = self._raw_events[event_channel_index].size
+        n = self._raw_events[event_channel_index][seg_index].size
         return n
 
     def _get_event_timestamps(self, block_index, seg_index, event_channel_index, t_start, t_stop):
 
-        raw_event = self._raw_events[event_channel_index]
+        raw_event = self._raw_events[event_channel_index][seg_index]
+
+        # important : all events timing are related to the first segment t_start
+        seg_start0, _ = self.info_segments[0]
 
         if t_start is not None:
-            keep = raw_event["start"] >= int(t_start * self._sampling_rate)
+            keep = raw_event["start"] + seg_start0 >= int(t_start * self._sampling_rate)
             raw_event = raw_event[keep]
 
         if t_stop is not None:
-            keep = raw_event["start"] <= int(t_stop * self._sampling_rate)
+            keep = raw_event["start"] + seg_start0 <= int(t_stop * self._sampling_rate)
             raw_event = raw_event[keep]
 
-        timestamp = raw_event["start"]
+        timestamp = raw_event["start"] + seg_start0
+
         if event_channel_index < 2:
             durations = None
         else: