Merge remote-tracking branch 'refs/remotes/origin/fix_openephys_stream' into fix_openephys_stream

h-mayorquin · h-mayorquin · commit 3f78992e16dd · 2025-03-21T10:35:27.000-06:00
diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml
@@ -10,57 +10,57 @@ on:
 
 jobs:
 
-  create-conda-env-cache-if-missing:
-    name: Caching conda env
-    runs-on: "ubuntu-latest"
-    strategy:
-      fail-fast: true
-    defaults:
-      # by default run in bash mode (required for conda usage)
-      run:
-        shell: bash -l {0}
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Get current year-month
-        id: date
-        run: |
-          echo "date=$(date +'%Y-%m')" >> $GITHUB_OUTPUT
-
-      - name: Get current dependencies hash
-        id: dependencies
-        run: |
-          echo "hash=${{hashFiles('**/pyproject.toml', '**/environment_testing.yml')}}" >> $GITHUB_OUTPUT
-
-      - uses: actions/cache@v3
-        # the cache for python package is reset:
-        #   * every month
-        #   * when package dependencies change
-        id: cache-conda-env
-        with:
-          path: /usr/share/miniconda/envs/neo-test-env
-          key: ${{ runner.os }}-conda-env-${{ steps.dependencies.outputs.hash }}-${{ steps.date.outputs.date }}
-
-      - name: Cache found?
-        run: echo "Cache-hit == ${{steps.cache-conda-env.outputs.cache-hit == 'true'}}"
-
-      # activate environment if not restored from cache
-      - uses: conda-incubator/setup-miniconda@v2.2.0
-        if: steps.cache-conda-env.outputs.cache-hit != 'true'
-        with:
-          activate-environment: neo-test-env
-          environment-file: environment_testing.yml
-          python-version: 3.9
-
-      - name: Create the conda environment to be cached
-        if: steps.cache-conda-env.outputs.cache-hit != 'true'
-        # create conda env, configure git and install pip, neo and test dependencies from master
-        # for PRs that change dependencies, this environment will be updated in the test workflow
-        run: |
-          git config --global user.email "neo_ci@fake_mail.com"
-          git config --global user.name "neo CI"
-          python -m pip install -U pip  # Official recommended way
-          pip install --upgrade -e .[test]
+  # create-conda-env-cache-if-missing:
+  #   name: Caching conda env
+  #   runs-on: "ubuntu-latest"
+  #   strategy:
+  #     fail-fast: true
+  #   defaults:
+  #     # by default run in bash mode (required for conda usage)
+  #     run:
+  #       shell: bash -l {0}
+  #   steps:
+  #     - uses: actions/checkout@v3
+
+  #     - name: Get current year-month
+  #       id: date
+  #       run: |
+  #         echo "date=$(date +'%Y-%m')" >> $GITHUB_OUTPUT
+
+  #     - name: Get current dependencies hash
+  #       id: dependencies
+  #       run: |
+  #         echo "hash=${{hashFiles('**/pyproject.toml', '**/environment_testing.yml')}}" >> $GITHUB_OUTPUT
+
+  #     - uses: actions/cache@v3
+  #       # the cache for python package is reset:
+  #       #   * every month
+  #       #   * when package dependencies change
+  #       id: cache-conda-env
+  #       with:
+  #         path: /usr/share/miniconda/envs/neo-test-env
+  #         key: ${{ runner.os }}-conda-env-${{ steps.dependencies.outputs.hash }}-${{ steps.date.outputs.date }}
+
+  #     - name: Cache found?
+  #       run: echo "Cache-hit == ${{steps.cache-conda-env.outputs.cache-hit == 'true'}}"
+
+  #     # activate environment if not restored from cache
+  #     - uses: conda-incubator/setup-miniconda@v2.2.0
+  #       if: steps.cache-conda-env.outputs.cache-hit != 'true'
+  #       with:
+  #         activate-environment: neo-test-env
+  #         environment-file: environment_testing.yml
+  #         python-version: 3.9
+
+      # - name: Create the conda environment to be cached
+      #   if: steps.cache-conda-env.outputs.cache-hit != 'true'
+      #   # create conda env, configure git and install pip, neo and test dependencies from master
+      #   # for PRs that change dependencies, this environment will be updated in the test workflow
+      #   run: |
+      #     git config --global user.email "neo_ci@fake_mail.com"
+      #     git config --global user.name "neo CI"
+      #     python -m pip install -U pip  # Official recommended way
+      #     pip install --upgrade -e .[test]
 
   create-data-cache-if-missing:
     name: Caching data env
diff --git a/doc/source/authors.rst b/doc/source/authors.rst
@@ -90,8 +90,10 @@ and may not be the current affiliation of a contributor.
 * Xin Niu
 * Nikhil Chandra [40]
 * Luigi Petrucco [42]
+* Tommaso Lambresa [43]
+* Nina Kudryashova [37]
 
-1. Centre de Recherche en Neuroscience de Lyon, CNRS UMR5292 - INSERM U1028 - Universite Claude Bernard Lyon 1
+1. Centre de Recherche en Neuroscience de Lyon, CNRS UMR5292 - INSERM U1028 - Université Claude Bernard Lyon 1
 2. Unité de Neuroscience, Information et Complexité, CNRS UPR 3293, Gif-sur-Yvette, France
 3. University of California, Berkeley
 4. Laboratoire de Neurosciences Intégratives et Adaptatives, CNRS UMR 6149 - Université de Provence, Marseille, France
@@ -133,6 +135,7 @@ and may not be the current affiliation of a contributor.
 40. Plexon Inc.
 41. Paris Brain Institute
 42. Istituto Italiano di Tecnologia (IIT), Italy
+43. University of Genoa, Italy
 
 
 
diff --git a/neo/io/__init__.py b/neo/io/__init__.py
@@ -167,7 +167,7 @@
 .. autoclass:: neo.io.MaxwellIO
 
     .. autoattribute:: extensions
-    
+
 .. autoclass:: neo.io.MedIO
 
     .. autoattribute:: extensions
diff --git a/neo/io/neomatlabio.py b/neo/io/neomatlabio.py
@@ -281,6 +281,8 @@ def write_block(self, bl, **kargs):
         """
         Arguments:
             bl: the block to be saved
+            kargs: extra keyword arguments broadcasted to scipy.io.savemat
+
         """
         import scipy.io
 
@@ -307,7 +309,7 @@ def write_block(self, bl, **kargs):
                     else:
                         group_structure[container_name].append(id(child_obj))
 
-        scipy.io.savemat(self.filename, {"block": bl_struct}, oned_as="row")
+        scipy.io.savemat(self.filename, {"block": bl_struct}, oned_as="row", **kargs)
 
     def _get_matlab_value(self, ob, attrname):
         units = None
diff --git a/neo/rawio/axonrawio.py b/neo/rawio/axonrawio.py
@@ -23,7 +23,7 @@
 strings section:
 [uModifierNameIndex, uCreatorNameIndex, uProtocolPathIndex, lFileComment, lADCCChannelNames, lADCUnitsIndex
 lDACChannelNameIndex, lDACUnitIndex, lDACFilePath, nLeakSubtractADC]
-['', 'Clampex', '', 'C:/path/protocol.pro', 'some comment', 'IN 0', 'mV', 'IN 1', 'mV', 'Cmd 0', 'pA', 
+['', 'Clampex', '', 'C:/path/protocol.pro', 'some comment', 'IN 0', 'mV', 'IN 1', 'mV', 'Cmd 0', 'pA',
 'Cmd 1', 'pA', 'Cmd 2', 'mV', 'Cmd 3', 'mV']
 
 Information on abf 1 and 2 formats is available here:
diff --git a/neo/rawio/blackrockrawio.py b/neo/rawio/blackrockrawio.py
@@ -979,7 +979,7 @@ def __read_nsx_dataheader_variant_b(
             # use of `int` avoids overflow problem
             data_size = int(dh["nb_data_points"]) * int(self.__nsx_basic_header[nsx_nb]["channel_count"]) * 2
             # define new offset (to possible next data block)
-            offset = data_header[index]["offset_to_data_block"] + data_size
+            offset = int(data_header[index]["offset_to_data_block"]) + data_size
 
             index += 1
 
diff --git a/neo/rawio/edfrawio.py b/neo/rawio/edfrawio.py
@@ -100,7 +100,8 @@ def _parse_header(self):
         for ch_idx, sig_dict in enumerate(self.signal_headers):
             ch_name = sig_dict["label"]
             chan_id = ch_idx
-            sr = sig_dict["sample_rate"]  # Hz
+            # pyedf >= 0.1.39 uses sample_frequency, pyedf < 0.1.39 uses sample_rate
+            sr = sig_dict.get("sample_frequency") or sig_dict.get("sample_rate")  # Hz
             dtype = "int16"  # assume general int16 based on edf documentation
             units = sig_dict["dimension"]
             physical_range = sig_dict["physical_max"] - sig_dict["physical_min"]
@@ -160,6 +161,7 @@ def _parse_header(self):
             "label",
             "dimension",
             "sample_rate",
+            "sample_frequency",
             "physical_min",
             "physical_max",
             "digital_min",
diff --git a/neo/rawio/intanrawio.py b/neo/rawio/intanrawio.py
@@ -16,7 +16,7 @@
   * http://intantech.com/files/Intan_RHD2000_data_file_formats.pdf
   * http://intantech.com/files/Intan_RHS2000_data_file_formats.pdf
 
-  
+
 Author: Samuel Garcia (Initial), Zach McKenzie & Heberto Mayorquin (Updates)
 
 """
diff --git a/neo/rawio/medrawio.py b/neo/rawio/medrawio.py
@@ -1,6 +1,6 @@
 """
 Class for reading MED (Multiscale Electrophysiology Data) Format.
-    
+
 Uses the dhn-med-py python package, created by Dark Horse Neuro, Inc.
 
 Authors: Dan Crepeau, Matt Stead
diff --git a/neo/rawio/neuronexusrawio.py b/neo/rawio/neuronexusrawio.py
@@ -6,13 +6,13 @@
   * The *.xdat.json metadata file
   * The *_data.xdat binary file of all raw data
   * The *_timestamps.xdat binary file of the timestamp data
-  
+
 Based on sample data is appears that the binary file is always a float32 format
 Other information can be found within the metadata json file
 
 
 The metadata file has a pretty complicated structure as far as I can tell
-a lot of which is dedicated to probe information, which won't be handle at the 
+a lot of which is dedicated to probe information, which won't be handle at the
 the Neo level.
 
 It appears that the metadata['status'] provides most of the information necessary
diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py
@@ -363,16 +363,14 @@ def _parse_header(self):
                                     rising_indices.extend(rising)
                                     falling_indices.extend(falling)
 
-                            rising_indices = np.array(rising_indices)
-                            falling_indices = np.array(falling_indices)
+                            rising_indices = np.array(rising_indices, dtype=np.int64)
+                            falling_indices = np.array(falling_indices, dtype=np.int64)
 
                             # Sort the indices to maintain chronological order
                             sorted_order = np.argsort(rising_indices)
                             rising_indices = rising_indices[sorted_order]
                             falling_indices = falling_indices[sorted_order]
 
-                            durations = None
-                            # if len(rising_indices) == len(falling_indices):
                             durations = timestamps[falling_indices] - timestamps[rising_indices]
                             if not self._use_direct_evt_timestamps:
                                 timestamps = timestamps / info["sample_rate"]
diff --git a/neo/rawio/tdtrawio.py b/neo/rawio/tdtrawio.py
@@ -205,28 +205,33 @@ def _parse_header(self):
         keep = info_channel_groups["TankEvType"] & EVTYPE_MASK == EVTYPE_STREAM
         missing_sev_channels = []
         for stream_index, info in enumerate(info_channel_groups[keep]):
+            stream_index = int(stream_index)  # This transforms numpy scalar to python native int  
             self._sig_sample_per_chunk[stream_index] = info["NumPoints"]
 
-            stream_name = str(info["StoreName"])
+            stream_name_bytes = info["StoreName"]
+            stream_name = info["StoreName"].decode("utf-8")
             stream_id = f"{stream_index}"
             buffer_id = ""
             signal_streams.append((stream_name, stream_id, buffer_id))
 
-            for c in range(info["NumChan"]):
+            for channel_index in range(info["NumChan"]):
                 global_chan_index = len(signal_channels)
-                chan_id = c + 1  # several StoreName can have same chan_id: this is ok
+                chan_id = channel_index + 1
 
                 # loop over segment to get sampling_rate/data_index/data_buffer
                 sampling_rate = None
                 dtype = None
                 for seg_index, segment_name in enumerate(segment_names):
                     # get data index
                     tsq = self._tsq[seg_index]
-                    mask = (
-                        (tsq["evtype"] & EVTYPE_MASK == EVTYPE_STREAM)
-                        & (tsq["evname"] == info["StoreName"])
-                        & (tsq["channel"] == chan_id)
-                    )
+                    # Filter TSQ events to find all data chunks belonging to the current stream and channel
+                    # This identifies which parts of the TEV/SEV files contain our signal data
+                    is_stream_event = (tsq["evtype"] & EVTYPE_MASK) == EVTYPE_STREAM  # Get only stream events (continuous data)
+                    matches_store_name = tsq["evname"] == stream_name_bytes  # Match the 4-char store name (e.g., 'RSn1')
+                    matches_channel = tsq["channel"] == chan_id  # Match the specific channel number
+
+                    # Combine all conditions - we want events that satisfy all three criteria
+                    mask = is_stream_event & matches_store_name & matches_channel
                     data_index = tsq[mask].copy()
                     self._sigs_index[seg_index][global_chan_index] = data_index
 
@@ -252,11 +257,11 @@ def _parse_header(self):
                     # sampling_rate and dtype
                     if len(data_index):
                         _sampling_rate = float(data_index["frequency"][0])
-                        _dtype = data_formats[data_index["dataformat"][0]]
+                        _dtype = data_formats_map[data_index["dataformat"][0]]
                     else:
                         # if no signal present use dummy values
                         _sampling_rate = 1.0
-                        _dtype = int
+                        _dtype = "int"
                     if sampling_rate is None:
                         sampling_rate = _sampling_rate
                         dtype = _dtype
@@ -277,17 +282,15 @@ def _parse_header(self):
                     # path = self.dirname / segment_name
                     if self.tdt_block_mode == "multi":
                         # for multi block datasets the names of sev files are fixed
-                        store = info["StoreName"].decode("ascii")
-                        sev_stem = f"{tankname}_{segment_name}_{store}_ch{chan_id}"
+                        sev_stem = f"{tankname}_{segment_name}_{stream_name}_ch{chan_id}"
                         sev_filename = (path / sev_stem).with_suffix(".sev")
                     else:
-                        # for single block datasets the exact name of sev files in not known
+                        # for single block datasets the exact name of sev files is not known
                         sev_regex = f"*_[cC]h{chan_id}.sev"
                         sev_filename = list(self.dirname.parent.glob(str(sev_regex)))
                         # in case multiple sev files are found, try to find the one for current stream
                         if len(sev_filename) > 1:
-                            store = info["StoreName"].decode("ascii")
-                            sev_regex = f"*_{store}_Ch{chan_id}.sev"
+                            sev_regex = f"*_{stream_name}_Ch{chan_id}.sev"
                             sev_filename = list(self.dirname.parent.glob(str(sev_regex)))
 
                         # in case non or multiple sev files are found for current stream + channel
@@ -305,14 +308,14 @@ def _parse_header(self):
                         raise NeoReadWriteError("no TEV nor SEV data to read")
                     self._sigs_data_buf[seg_index][global_chan_index] = data
 
-                chan_name = f"{info['StoreName']} {c + 1}"
+                channel_name = f"{stream_name} {channel_index + 1}"
                 sampling_rate = sampling_rate
                 units = "uV"  # see https://github.com/NeuralEnsemble/python-neo/issues/1369
                 gain = 1.0
                 offset = 0.0
                 buffer_id = ""
                 signal_channels.append(
-                    (chan_name, str(chan_id), sampling_rate, dtype, units, gain, offset, stream_id, buffer_id)
+                    (channel_name, str(chan_id), sampling_rate, dtype, units, gain, offset, stream_id, buffer_id)
                 )
 
         if missing_sev_channels:
@@ -354,7 +357,7 @@ def _parse_header(self):
                     )
 
                     self._waveforms_size.append(info["NumPoints"])
-                    self._waveforms_dtype.append(np.dtype(data_formats[info["DataFormat"]]))
+                    self._waveforms_dtype.append(np.dtype(data_formats_map[info["DataFormat"]]))
 
         spike_channels = np.array(spike_channels, dtype=_spike_channel_dtype)
 
@@ -417,13 +420,13 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, strea
         global_chan_indexes = global_chan_indexes[channel_indexes]
         signal_channels = signal_channels[channel_indexes]
 
-        dt = self._sig_dtype_by_group[stream_index]
-        raw_signals = np.zeros((i_stop - i_start, signal_channels.size), dtype=dt)
+        dtype = self._sig_dtype_by_group[stream_index]
+        raw_signals = np.zeros((i_stop - i_start, signal_channels.size), dtype=dtype)
 
         sample_per_chunk = self._sig_sample_per_chunk[stream_index]
         bl0 = i_start // sample_per_chunk
         bl1 = int(np.ceil(i_stop / sample_per_chunk))
-        chunk_nb_bytes = sample_per_chunk * dt.itemsize
+        chunk_nb_bytes = sample_per_chunk * dtype.itemsize
 
         for c, global_index in enumerate(global_chan_indexes):
             data_index = self._sigs_index[seg_index][global_index]
@@ -434,7 +437,7 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, strea
             for bl in range(bl0, bl1):
                 ind0 = data_index[bl]["offset"]
                 ind1 = ind0 + chunk_nb_bytes
-                data = data_buf[ind0:ind1].view(dt)
+                data = data_buf[ind0:ind1].view(dtype)
 
                 if bl == bl1 - 1:
                     # right border
@@ -620,7 +623,7 @@ def read_tbk(tbk_filename):
 EVMARK_STARTBLOCK = int("0001", 16)  # 1
 EVMARK_STOPBLOCK = int("0002", 16)  # 2
 
-data_formats = {
+data_formats_map = {
     0: "float32",
     1: "int32",
     2: "int16",
diff --git a/neo/test/iotest/test_openephysbinaryio.py b/neo/test/iotest/test_openephysbinaryio.py
@@ -1,6 +1,4 @@
-"""
-
-"""
+""" """
 
 import unittest
 
diff --git a/neo/test/iotest/test_openephysio.py b/neo/test/iotest/test_openephysio.py
@@ -1,6 +1,4 @@
-"""
-
-"""
+""" """
 
 import unittest
 
diff --git a/neo/test/rawiotest/test_brainvisionrawio.py b/neo/test/rawiotest/test_brainvisionrawio.py
diff --git a/neo/test/rawiotest/test_micromedrawio.py b/neo/test/rawiotest/test_micromedrawio.py
