Merge branch 'master' into authors

Author: zm711

neo/io/__init__.py

@@ -167,7 +167,7 @@
 .. autoclass:: neo.io.MaxwellIO
 
     .. autoattribute:: extensions
-    
+
 .. autoclass:: neo.io.MedIO
 
     .. autoattribute:: extensions

neo/rawio/axonrawio.py

@@ -23,7 +23,7 @@
 strings section:
 [uModifierNameIndex, uCreatorNameIndex, uProtocolPathIndex, lFileComment, lADCCChannelNames, lADCUnitsIndex
 lDACChannelNameIndex, lDACUnitIndex, lDACFilePath, nLeakSubtractADC]
-['', 'Clampex', '', 'C:/path/protocol.pro', 'some comment', 'IN 0', 'mV', 'IN 1', 'mV', 'Cmd 0', 'pA', 
+['', 'Clampex', '', 'C:/path/protocol.pro', 'some comment', 'IN 0', 'mV', 'IN 1', 'mV', 'Cmd 0', 'pA',
 'Cmd 1', 'pA', 'Cmd 2', 'mV', 'Cmd 3', 'mV']
 
 Information on abf 1 and 2 formats is available here:

neo/rawio/intanrawio.py

@@ -16,7 +16,7 @@
   * http://intantech.com/files/Intan_RHD2000_data_file_formats.pdf
   * http://intantech.com/files/Intan_RHS2000_data_file_formats.pdf
 
-  
+
 Author: Samuel Garcia (Initial), Zach McKenzie & Heberto Mayorquin (Updates)
 
 """
@@ -531,6 +531,57 @@ def _demultiplex_digital_data(self, raw_digital_data, channel_ids, i_start, i_st
 
         return output
 
+    def get_intan_timestamps(self, i_start=None, i_stop=None):
+        """
+        Retrieves the sample indices from the Intan raw data within a specified range.
+
+        Note that sample indices are called timestamps in the Intan format but they are
+        in fact just sample indices. This function extracts the sample index timestamps
+        from Intan files, which represent  relative time points in sample units (not absolute time).
+        These indices can be  particularly useful when working with recordings that have discontinuities.
+
+        Parameters
+        ----------
+        i_start : int, optional
+            The starting index from which to retrieve sample indices. If None, starts from 0.
+        i_stop : int, optional
+            The stopping index up to which to retrieve sample indices (exclusive).
+            If None, retrieves all available indices from i_start onward.
+
+        Returns
+        -------
+        timestamps : ndarray
+            The flattened array of sample indices within the specified range.
+
+        Notes
+        -----
+        - Sample indices can be converted to seconds by dividing by the sampling rate of the amplifier stream.
+        - The function automatically handles different file formats:
+        * header-attached: Timestamps are extracted directly from the timestamp field
+        * one-file-per-signal: Timestamps are read from the timestamp stream
+        * one-file-per-channel: Timestamps are read from the first channel in the timestamp stream
+        - When recordings have discontinuities (indicated by the `discontinuous_timestamps`
+        attribute being True), these indices allow for proper temporal alignment of the data.
+        """
+        if i_start is None:
+            i_start = 0
+
+        # Get the timestamps based on file format
+        if self.file_format == "header-attached":
+            timestamps = self._raw_data["timestamp"]
+        elif self.file_format == "one-file-per-signal":
+            timestamps = self._raw_data["timestamp"]
+        elif self.file_format == "one-file-per-channel":
+            timestamps = self._raw_data["timestamp"][0]
+
+        # TODO if possible ensure that timestamps memmaps are always of correct shape to avoid memory copy here.
+        timestamps = timestamps.flatten() if timestamps.ndim > 1 else timestamps
+
+        if i_stop is None:
+            return timestamps[i_start:]
+        else:
+            return timestamps[i_start:i_stop]
+
     def _assert_timestamp_continuity(self):
         """
         Asserts the continuity of timestamps in the data.
@@ -545,26 +596,11 @@ def _assert_timestamp_continuity(self):
         NeoReadWriteError
             If timestamps are not continuous and `ignore_integrity_checks` is False.
             The error message includes a table detailing the discontinuities found.
-
-        Notes
-        -----
-        The method extracts timestamps from the raw data based on the file format:
-
-        * **header-attached:** Timestamps are extracted from a 'timestamp' field in the raw data.
-        * **one-file-per-signal:** Timestamps are taken from the last stream.
-        * **one-file-per-channel:** Timestamps are retrieved from the first channel of the last stream.
         """
         # check timestamp continuity
-        if self.file_format == "header-attached":
-            timestamp = self._raw_data["timestamp"].flatten()
-
-        # timestamps are always last stream for headerless binary files
-        elif self.file_format == "one-file-per-signal":
-            timestamp = self._raw_data["timestamp"]
-        elif self.file_format == "one-file-per-channel":
-            timestamp = self._raw_data["timestamp"][0]
+        timestamps = self.get_intan_timestamps()
 
-        discontinuous_timestamps = np.diff(timestamp) != 1
+        discontinuous_timestamps = np.diff(timestamps) != 1
         timestamps_are_not_contiguous = np.any(discontinuous_timestamps)
         if timestamps_are_not_contiguous:
             # Mark a flag that can be checked after parsing the header to see if the timestamps are continuous or not
@@ -582,8 +618,8 @@ def _assert_timestamp_continuity(self):
 
                 amplifier_sampling_rate = self._global_info["sampling_rate"]
                 for discontinuity_index in np.where(discontinuous_timestamps)[0]:
-                    prev_ts = timestamp[discontinuity_index]
-                    next_ts = timestamp[discontinuity_index + 1]
+                    prev_ts = timestamps[discontinuity_index]
+                    next_ts = timestamps[discontinuity_index + 1]
                     time_diff = (next_ts - prev_ts) / amplifier_sampling_rate
 
                     error_msg += (

neo/rawio/medrawio.py

@@ -1,6 +1,6 @@
 """
 Class for reading MED (Multiscale Electrophysiology Data) Format.
-    
+
 Uses the dhn-med-py python package, created by Dark Horse Neuro, Inc.
 
 Authors: Dan Crepeau, Matt Stead

neo/rawio/neuronexusrawio.py

@@ -6,13 +6,13 @@
   * The *.xdat.json metadata file
   * The *_data.xdat binary file of all raw data
   * The *_timestamps.xdat binary file of the timestamp data
-  
+
 Based on sample data is appears that the binary file is always a float32 format
 Other information can be found within the metadata json file
 
 
 The metadata file has a pretty complicated structure as far as I can tell
-a lot of which is dedicated to probe information, which won't be handle at the 
+a lot of which is dedicated to probe information, which won't be handle at the
 the Neo level.
 
 It appears that the metadata['status'] provides most of the information necessary

neo/rawio/openephysbinaryrawio.py

@@ -292,16 +292,14 @@ def _parse_header(self):
                                     rising_indices.extend(rising)
                                     falling_indices.extend(falling)
 
-                            rising_indices = np.array(rising_indices)
-                            falling_indices = np.array(falling_indices)
+                            rising_indices = np.array(rising_indices, dtype=np.int64)
+                            falling_indices = np.array(falling_indices, dtype=np.int64)
 
                             # Sort the indices to maintain chronological order
                             sorted_order = np.argsort(rising_indices)
                             rising_indices = rising_indices[sorted_order]
                             falling_indices = falling_indices[sorted_order]
 
-                            durations = None
-                            # if len(rising_indices) == len(falling_indices):
                             durations = timestamps[falling_indices] - timestamps[rising_indices]
                             if not self._use_direct_evt_timestamps:
                                 timestamps = timestamps / info["sample_rate"]

neo/rawio/tdtrawio.py

@@ -205,28 +205,37 @@ def _parse_header(self):
         keep = info_channel_groups["TankEvType"] & EVTYPE_MASK == EVTYPE_STREAM
         missing_sev_channels = []
         for stream_index, info in enumerate(info_channel_groups[keep]):
+            stream_index = int(stream_index)  # This transforms numpy scalar to python native int
             self._sig_sample_per_chunk[stream_index] = info["NumPoints"]
 
-            stream_name = str(info["StoreName"])
+            stream_name_bytes = info["StoreName"]
+            stream_name = info["StoreName"].decode("utf-8")
             stream_id = f"{stream_index}"
             buffer_id = ""
             signal_streams.append((stream_name, stream_id, buffer_id))
 
-            for c in range(info["NumChan"]):
+            for channel_index in range(info["NumChan"]):
                 global_chan_index = len(signal_channels)
-                chan_id = c + 1  # several StoreName can have same chan_id: this is ok
+                chan_id = channel_index + 1
 
                 # loop over segment to get sampling_rate/data_index/data_buffer
                 sampling_rate = None
                 dtype = None
                 for seg_index, segment_name in enumerate(segment_names):
                     # get data index
                     tsq = self._tsq[seg_index]
-                    mask = (
-                        (tsq["evtype"] & EVTYPE_MASK == EVTYPE_STREAM)
-                        & (tsq["evname"] == info["StoreName"])
-                        & (tsq["channel"] == chan_id)
-                    )
+                    # Filter TSQ events to find all data chunks belonging to the current stream and channel
+                    # This identifies which parts of the TEV/SEV files contain our signal data
+                    is_stream_event = (
+                        tsq["evtype"] & EVTYPE_MASK
+                    ) == EVTYPE_STREAM  # Get only stream events (continuous data)
+                    matches_store_name = (
+                        tsq["evname"] == stream_name_bytes
+                    )  # Match the 4-char store name (e.g., 'RSn1')
+                    matches_channel = tsq["channel"] == chan_id  # Match the specific channel number
+
+                    # Combine all conditions - we want events that satisfy all three criteria
+                    mask = is_stream_event & matches_store_name & matches_channel
                     data_index = tsq[mask].copy()
                     self._sigs_index[seg_index][global_chan_index] = data_index
 
@@ -252,11 +261,11 @@ def _parse_header(self):
                     # sampling_rate and dtype
                     if len(data_index):
                         _sampling_rate = float(data_index["frequency"][0])
-                        _dtype = data_formats[data_index["dataformat"][0]]
+                        _dtype = data_formats_map[data_index["dataformat"][0]]
                     else:
                         # if no signal present use dummy values
                         _sampling_rate = 1.0
-                        _dtype = int
+                        _dtype = "int"
                     if sampling_rate is None:
                         sampling_rate = _sampling_rate
                         dtype = _dtype
@@ -277,17 +286,15 @@ def _parse_header(self):
                     # path = self.dirname / segment_name
                     if self.tdt_block_mode == "multi":
                         # for multi block datasets the names of sev files are fixed
-                        store = info["StoreName"].decode("ascii")
-                        sev_stem = f"{tankname}_{segment_name}_{store}_ch{chan_id}"
+                        sev_stem = f"{tankname}_{segment_name}_{stream_name}_ch{chan_id}"
                         sev_filename = (path / sev_stem).with_suffix(".sev")
                     else:
-                        # for single block datasets the exact name of sev files in not known
+                        # for single block datasets the exact name of sev files is not known
                         sev_regex = f"*_[cC]h{chan_id}.sev"
                         sev_filename = list(self.dirname.parent.glob(str(sev_regex)))
                         # in case multiple sev files are found, try to find the one for current stream
                         if len(sev_filename) > 1:
-                            store = info["StoreName"].decode("ascii")
-                            sev_regex = f"*_{store}_Ch{chan_id}.sev"
+                            sev_regex = f"*_{stream_name}_Ch{chan_id}.sev"
                             sev_filename = list(self.dirname.parent.glob(str(sev_regex)))
 
                         # in case non or multiple sev files are found for current stream + channel
@@ -305,14 +312,14 @@ def _parse_header(self):
                         raise NeoReadWriteError("no TEV nor SEV data to read")
                     self._sigs_data_buf[seg_index][global_chan_index] = data
 
-                chan_name = f"{info['StoreName']} {c + 1}"
+                channel_name = f"{stream_name} {channel_index + 1}"
                 sampling_rate = sampling_rate
                 units = "uV"  # see https://github.com/NeuralEnsemble/python-neo/issues/1369
                 gain = 1.0
                 offset = 0.0
                 buffer_id = ""
                 signal_channels.append(
-                    (chan_name, str(chan_id), sampling_rate, dtype, units, gain, offset, stream_id, buffer_id)
+                    (channel_name, str(chan_id), sampling_rate, dtype, units, gain, offset, stream_id, buffer_id)
                 )
 
         if missing_sev_channels:
@@ -354,7 +361,7 @@ def _parse_header(self):
                     )
 
                     self._waveforms_size.append(info["NumPoints"])
-                    self._waveforms_dtype.append(np.dtype(data_formats[info["DataFormat"]]))
+                    self._waveforms_dtype.append(np.dtype(data_formats_map[info["DataFormat"]]))
 
         spike_channels = np.array(spike_channels, dtype=_spike_channel_dtype)
 
@@ -417,13 +424,13 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, strea
         global_chan_indexes = global_chan_indexes[channel_indexes]
         signal_channels = signal_channels[channel_indexes]
 
-        dt = self._sig_dtype_by_group[stream_index]
-        raw_signals = np.zeros((i_stop - i_start, signal_channels.size), dtype=dt)
+        dtype = self._sig_dtype_by_group[stream_index]
+        raw_signals = np.zeros((i_stop - i_start, signal_channels.size), dtype=dtype)
 
         sample_per_chunk = self._sig_sample_per_chunk[stream_index]
         bl0 = i_start // sample_per_chunk
         bl1 = int(np.ceil(i_stop / sample_per_chunk))
-        chunk_nb_bytes = sample_per_chunk * dt.itemsize
+        chunk_nb_bytes = sample_per_chunk * dtype.itemsize
 
         for c, global_index in enumerate(global_chan_indexes):
             data_index = self._sigs_index[seg_index][global_index]
@@ -434,7 +441,7 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, strea
             for bl in range(bl0, bl1):
                 ind0 = data_index[bl]["offset"]
                 ind1 = ind0 + chunk_nb_bytes
-                data = data_buf[ind0:ind1].view(dt)
+                data = data_buf[ind0:ind1].view(dtype)
 
                 if bl == bl1 - 1:
                     # right border
@@ -620,7 +627,7 @@ def read_tbk(tbk_filename):
 EVMARK_STARTBLOCK = int("0001", 16)  # 1
 EVMARK_STOPBLOCK = int("0002", 16)  # 2
 
-data_formats = {
+data_formats_map = {
     0: "float32",
     1: "int32",
     2: "int16",

neo/test/iotest/test_openephysbinaryio.py

@@ -1,6 +1,4 @@
-"""
-
-"""
+""" """
 
 import unittest
 

neo/test/iotest/test_openephysio.py

@@ -1,6 +1,4 @@
-"""
-
-"""
+""" """
 
 import unittest
 

neo/test/rawiotest/test_brainvisionrawio.py

@@ -1,6 +1,4 @@
-"""
-
-"""
+""" """
 
 import unittest