Merge branch 'master' into fix_blackrock

Author: h-mayorquin

.github/workflows/io-test.yml

@@ -114,13 +114,6 @@ jobs:
       #  run: |
       #    pip install --no-dependencies -e .
 
-      - name: Install wine
-        run: |
-          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
-          sudo dpkg --add-architecture i386
-          sudo apt-get update -qq
-          sudo apt-get install -yqq --allow-downgrades libc6:i386 libgcc-s1:i386 libstdc++6:i386 wine
-
       - name: Pip list
         run: |
           pip list

.github/workflows/plexon2-testing.yml

@@ -0,0 +1,109 @@
+name: NeoPlexon2Test
+
+on:
+  workflow_dispatch:
+    schedule:
+      - cron: "0 12 * * 0" # weekly at noon UTC on Sundays
+
+
+jobs:
+  build-and-test:
+    name: Test on (${{ inputs.os }}) (${{ matrix.python-version}}) (${{ matrix.numpy-version }})
+    runs-on: ${{ inputs.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ['3.9', '3.12']
+        numpy-version: ['1.26', '2.0']
+    defaults:
+      # by default run in bash mode (required for conda usage)
+      run:
+        shell: bash -l {0}
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Get current year-month
+        id: date
+        run: echo "date=$(date +'%Y-%m')" >> $GITHUB_OUTPUT
+
+      - name: Get ephy_testing_data current head hash
+        # the key depend on the last commit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git
+        id: ephy_testing_data
+        run: |
+          echo "dataset_hash=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache/restore@v4
+        # Loading cache of ephys_testing_dataset
+        id: cache-datasets
+        with:
+          path: ~/ephy_testing_data
+          key: ${{ runner.os }}-datasets-${{ steps.ephy_testing_data.outputs.dataset_hash }}
+          restore-keys: ${{ runner.os }}-datasets-
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: neo-test-env-${{ matrix.python-version }}
+          python-version: "${{ matrix.python-version }}"
+
+      - name: Install testing dependencies
+        # testing environment is only created from yml if no cache was found
+        # restore-key hits should result in `cache-hit` == 'false'
+        #if: steps.cache-conda-env.outputs.cache-hit != 'true'
+        run: |
+          conda install pip numpy=${{ matrix.numpy-version }} -c conda-forge
+        # this command is for updating cache. We are resting removal.
+        # conda env update --name neo-test-env-${{ matrix.python-version }} --file environment_testing.yml --prune
+
+      - name: Install git-annex
+        # this is the trick from the spikeinterface repo for getting git-annex to work with datalad
+        # see https://github.com/SpikeInterface/spikeinterface/pull/3877 for more info
+        shell: bash
+        run: |
+          pip install datalad-installer
+          datalad-installer --sudo ok git-annex --method datalad/packages
+          git config --global filter.annex.process "git-annex filter-process"  # recommended for efficiency
+
+      - name: Configure git
+        run: |
+          git config --global user.email "neo_ci@fake_mail.com"
+          git config --global user.name "neo CI"
+
+      - name: Python version
+        run: |
+          which python
+          python --version
+
+      - name: Install neo including dependencies
+        # installation with dependencies is only required if no cache was found
+        # restore-key hits should result in `cache-hit` == 'false'
+        # if: steps.cache-conda-env.outputs.cache-hit != 'true'
+        run: |
+          pip install --upgrade -e .
+          pip install .[test]
+
+
+      - name: Install wine
+        run: |
+          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
+          sudo dpkg --add-architecture i386
+          sudo apt-get update -qq
+          sudo apt-get install -yqq --allow-downgrades libc6:i386 libgcc-s1:i386 libstdc++6:i386 wine
+
+      - name: Pip list
+        run: |
+          pip list
+
+      - name: Conda list
+        run: |
+          conda list
+
+      - name: Test with pytest
+        env:
+          HDF5_PLUGIN_PATH: ${{ github.workspace }}/hdf5_local_plugin_path
+          PLEXON2_TEST: true
+        run: |
+          # only neo.rawio and neo.io
+          pytest --cov=neo neo/test/rawiotest
+          pytest --cov=neo neo/test/iotest

doc/source/authors.rst

@@ -71,7 +71,7 @@ and may not be the current affiliation of a contributor.
 * Daniel P. Crepeau [30]
 * Divyansh Gupta [31]
 * Nate Dolensek [3]
-* Philipp Hornauer [32]
+* Philipp Hornauer [32, 45]
 * Robert Wolff [42]
 * Jules Lebert [33]
 * Benjamin Heasly
@@ -95,6 +95,9 @@ and may not be the current affiliation of a contributor.
 * Rémi Proville [44]
 * Paul Adkisson [24]
 * Luiz Tauffer [24]
+* Akshaj Verma [46]
+* Letizia Signorelli [47]
+* Daniel Parthier [48]
 
 1. Centre de Recherche en Neuroscience de Lyon, CNRS UMR5292 - INSERM U1028 - Université Claude Bernard Lyon 1
 2. Unité de Neuroscience, Information et Complexité, CNRS UPR 3293, Gif-sur-Yvette, France
@@ -140,6 +143,10 @@ and may not be the current affiliation of a contributor.
 42. Istituto Italiano di Tecnologia (IIT), Genoa, Italy
 43. University of Genoa, Italy
 44. AquiNeuro, SAS
+45. Maxwell Biosystems AG
+46. Brain Center, University Medical Center Utrecht, Utrecht University, The Netherlands
+47. Centre for Molecular Medicine Norway (NCMM), University of Oslo, Norway
+48. Charité - Universitätmedizin Berlin, Freie Universität Berlin and Humboldt Universität zu Berlin
 
 
 

doc/source/governance.rst

@@ -38,6 +38,7 @@ The current maintainers are:
 - Julia Sprenger (`@JuliaSprenger`_)
 - Michael Denker (`@mdenker`_)
 - Alessio Buccino (`@alejoe91`_)
+- Zach McKenzie (`@zm711`_)
 
 
 .. _`Neo maintainers team`: https://github.com/orgs/NeuralEnsemble/teams/neo-maintainers
@@ -47,3 +48,4 @@ The current maintainers are:
 .. _`@JuliaSprenger`: https://github.com/JuliaSprenger
 .. _`@mdenker`: https://github.com/mdenker
 .. _`@alejoe91`: https://github.com/alejoe91
+.. _`@zm711`: https://github.com/zm711

neo/core/spiketrainlist.py

@@ -113,8 +113,11 @@ def __init__(self, items=None, parent=None):
         self._channel_id_array = None
         self._all_channel_ids = None
         self._spiketrain_metadata = {}
-        if parent is not None and parent.__class__.__name__ != "Segment":
-            raise AttributeError("The parent class must be a Segment")
+
+        from .segment import Segment  # import here rather than at the top to avoid circular import
+
+        if parent is not None and not isinstance(parent, Segment):
+            raise AttributeError("If provided, the parent class must be a Segment")
         self.segment = parent
 
     @property

neo/rawio/axonarawio.py

@@ -377,8 +377,7 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, strea
         if channel_indexes is None:
             channel_indexes = [i for i in range(bin_dict["num_channels"])]
         elif isinstance(channel_indexes, slice):
-            channel_indexes_all = [i for i in range(bin_dict["num_channels"])]
-            channel_indexes = channel_indexes_all[channel_indexes]
+            channel_indexes = self._get_active_channels()
 
         num_samples = i_stop - i_start
 
@@ -562,6 +561,20 @@ def get_active_tetrode(self):
                     active_tetrodes.append(tetrode_id)
         return active_tetrodes
 
+    def _get_active_channels(self):
+        """
+        Returns the ID numbers of the active channels as a list.
+        E.g.: [20,21,22,23] for tetrode 6 active.
+        """
+        active_tetrodes = self.get_active_tetrode()
+        active_channels = []
+
+        for tetrode in active_tetrodes:
+            chans = self._get_channel_from_tetrode(tetrode)
+            active_channels.append(chans)
+
+        return np.concatenate(active_channels)
+
     def _get_channel_from_tetrode(self, tetrode):
         """
         This function will take the tetrode number and return the Axona
@@ -632,12 +645,13 @@ def _get_signal_chan_header(self):
         gain_list = self._get_channel_gain()
         offset = 0  # What is the offset?
 
+        first_channel = (active_tetrode_set[0] - 1) * elec_per_tetrode
         sig_channels = []
         for itetr in range(num_active_tetrode):
 
             for ielec in range(elec_per_tetrode):
-                cntr = (itetr * elec_per_tetrode) + ielec
-                ch_name = f"{itetr + 1}{letters[ielec]}"
+                cntr = (itetr * elec_per_tetrode) + ielec + first_channel
+                ch_name = f"{itetr + active_tetrode_set[0]}{letters[ielec]}"
                 chan_id = str(cntr)
                 gain = gain_list[cntr]
                 stream_id = "0"

neo/rawio/blackrockrawio.py

@@ -605,10 +605,17 @@ def _parse_header(self):
             for c in range(spike_channels.size):
                 st_ann = seg_ann["spikes"][c]
                 channel_id, unit_id = self.internal_unit_ids[c]
-                unit_tag = {0: "unclassified", 255: "noise"}.get(unit_id, str(unit_id))
                 st_ann["channel_id"] = channel_id
                 st_ann["unit_id"] = unit_id
-                st_ann["unit_tag"] = unit_tag
+                if unit_id == 0:
+                    st_ann["unit_classification"] = "unclassified"
+                elif 1 <= unit_id <= 16:
+                    st_ann["unit_classification"] = "sorted"
+                elif unit_id == 255:
+                    st_ann["unit_classification"] = "noise"
+                else:  # 17-254 are reserved
+                    st_ann["unit_classification"] = "reserved"
+                st_ann["unit_tag"] = st_ann["unit_classification"]
                 st_ann["description"] = f"SpikeTrain channel_id: {channel_id}, unit_id: {unit_id}"
                 st_ann["file_origin"] = self._filenames["nev"] + ".nev"
 
@@ -1273,7 +1280,19 @@ def __read_nev_data(self, nev_data_masks, nev_data_types):
         # read all raw data packets and markers
         dt0 = [("timestamp", ts_format), ("packet_id", "uint16"), ("value", f"S{data_size - header_skip}")]
 
-        raw_data = np.memmap(filename, offset=header_size, dtype=dt0, mode="r")
+        # expected number of data packets. We are not sure why, but it seems we can get partial data packets
+        # based on blackrock's own code this is okay so applying an int to round down is necessary to obtain the
+        # memory map of full packets and toss the partial packet.
+        # See reference: https://github.com/BlackrockNeurotech/Python-Utilities/blob/fa75aa671680306788e10d3d8dd625f9da4ea4f6/brpylib/brpylib.py#L580-L587
+        n_packets = int((self.__get_file_size(filename) - header_size) / data_size)
+
+        raw_data = np.memmap(
+            filename,
+            offset=header_size,
+            dtype=dt0,
+            shape=(n_packets,),
+            mode="r",
+        )
 
         masks = self.__nev_data_masks(raw_data["packet_id"])
         types = self.__nev_data_types(data_size)
@@ -1816,7 +1835,7 @@ def __nev_params(self, param_name):
                 hour=self.__nev_basic_header["hour"],
                 minute=self.__nev_basic_header["minute"],
                 second=self.__nev_basic_header["second"],
-                microsecond=self.__nev_basic_header["millisecond"],
+                microsecond=int(self.__nev_basic_header["millisecond"]) * 1000,
             ),
             "max_res": self.__nev_basic_header["timestamp_resolution"],
             "channel_ids": self.__nev_ext_header[b"NEUEVWAV"]["electrode_id"],

neo/rawio/maxwellrawio.py

@@ -242,18 +242,20 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, strea
             raise (e)
 
 
-_hdf_maxwell_error = """Maxwell file format is based on HDF5.
-The internal compression requires a custom plugin!!!
-This is a big pain for the end user.
-You, as a end user, should ask Maxwell company to change this.
-Please visit this page and install the missing decompression libraries:
-https://share.mxwbio.com/d/4742248b2e674a85be97/
-Then, link the decompression library by setting the `HDF5_PLUGIN_PATH` to your
-installation location, e.g. via
+_hdf_maxwell_error = """The MaxWell file compression requires a custom plugin.
+You can use the auto_install_maxwell_hdf5_compression_plugin() function or 
+(if it fails) install it manually:
+Download the missing decompression library:
+https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35
+Then, link the decompression library by setting the `HDF5_PLUGIN_PATH` to its location, 
+e.g. directly in Python via:
 os.environ['HDF5_PLUGIN_PATH'] = '/path/to/custom/hdf5/plugin/'
-
-Alternatively, you can use the auto_install_maxwell_hdf5_compression_plugin() below
-function that do it automagically.
+or in your shell via:
+export HDF5_PLUGIN_PATH=/path/to/custom/hdf5/plugin/
+You can also set the `HDF5_PLUGIN_PATH` environment variable in your shell
+configuration file (e.g. .bashrc, .bash_profile, .zshrc, etc.) to make it
+permanent. 
+See https://mxw.bio/MxW_Doc_Installing_Decompression_Library_to_load_MaxLab_Live_Recordings for more details.
 """
 
 
@@ -267,13 +269,17 @@ def auto_install_maxwell_hdf5_compression_plugin(hdf5_plugin_path=None, force_do
     hdf5_plugin_path.mkdir(exist_ok=True)
 
     if platform.system() == "Linux":
-        remote_lib = "https://share.mxwbio.com/d/4742248b2e674a85be97/files/?p=%2FLinux%2Flibcompression.so&dl=1"
+        remote_lib = "https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35/files/?p=%2FLinux%2Flibcompression.so&dl=1"
         local_lib = hdf5_plugin_path / "libcompression.so"
     elif platform.system() == "Darwin":
-        remote_lib = "https://share.mxwbio.com/d/4742248b2e674a85be97/files/?p=%2FMacOS%2Flibcompression.dylib&dl=1"
+        if platform.machine() == "arm64":
+            remote_lib = "https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35/files/?p=%2FMacOS%2FMac_arm64%2Flibcompression.dylib&dl=1"
+        else:
+            # Assuming x86_64 for MacOS
+            remote_lib = "https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35/files/?p=%2FMacOS%2FMac_x86_64%2Flibcompression.dylib&dl=1"
         local_lib = hdf5_plugin_path / "libcompression.dylib"
     elif platform.system() == "Windows":
-        remote_lib = "https://share.mxwbio.com/d/4742248b2e674a85be97/files/?p=%2FWindows%2Fcompression.dll&dl=1"
+        remote_lib = "https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35/files/?p=%2FWindows%2Fcompression.dll&dl=1"
         local_lib = hdf5_plugin_path / "compression.dll"
 
     if not force_download and local_lib.is_file():

neo/rawio/micromedrawio.py

@@ -51,7 +51,6 @@ def __init__(self, filename=""):
         self.filename = filename
 
     def _parse_header(self):
-
         with open(self.filename, "rb") as fid:
             f = StructFile(fid)
 
@@ -133,7 +132,11 @@ def _parse_header(self):
             sig_grounds = []
             for c in range(Num_Chan):
                 zname2, pos, length = zones["LABCOD"]
-                f.seek(pos + code[c] * 128 + 2, 0)
+                # Force code[c] which is currently a uint16 (or u2) into a int to prevent integer overflow
+                # for the following operation -- code[c] * 128 + 2.
+                # An integer overflow below may have side - effects including but not limited
+                # to having repeated channel names.
+                f.seek(pos + int(code[c]) * 128 + 2, 0)
 
                 chan_name = f.read(6).strip(b"\x00").decode("ascii")
                 ground = f.read(6).strip(b"\x00").decode("ascii")
@@ -269,7 +272,6 @@ def _event_count(self, block_index, seg_index, event_channel_index):
         return n
 
     def _get_event_timestamps(self, block_index, seg_index, event_channel_index, t_start, t_stop):
-
         raw_event = self._raw_events[event_channel_index][seg_index]
 
         # important : all events timing are related to the first segment t_start