Merge branch 'master' into micromed

Author: zm711

.github/workflows/io-test.yml

@@ -114,13 +114,6 @@ jobs:
       #  run: |
       #    pip install --no-dependencies -e .
 
-      - name: Install wine
-        run: |
-          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
-          sudo dpkg --add-architecture i386
-          sudo apt-get update -qq
-          sudo apt-get install -yqq --allow-downgrades libc6:i386 libgcc-s1:i386 libstdc++6:i386 wine
-
       - name: Pip list
         run: |
           pip list

.github/workflows/plexon2-testing.yml

@@ -0,0 +1,109 @@
+name: NeoPlexon2Test
+
+on:
+  workflow_dispatch:
+    schedule:
+      - cron: "0 12 * * 0" # weekly at noon UTC on Sundays
+
+
+jobs:
+  build-and-test:
+    name: Test on (${{ inputs.os }}) (${{ matrix.python-version}}) (${{ matrix.numpy-version }})
+    runs-on: ${{ inputs.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ['3.9', '3.12']
+        numpy-version: ['1.26', '2.0']
+    defaults:
+      # by default run in bash mode (required for conda usage)
+      run:
+        shell: bash -l {0}
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Get current year-month
+        id: date
+        run: echo "date=$(date +'%Y-%m')" >> $GITHUB_OUTPUT
+
+      - name: Get ephy_testing_data current head hash
+        # the key depend on the last commit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git
+        id: ephy_testing_data
+        run: |
+          echo "dataset_hash=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache/restore@v4
+        # Loading cache of ephys_testing_dataset
+        id: cache-datasets
+        with:
+          path: ~/ephy_testing_data
+          key: ${{ runner.os }}-datasets-${{ steps.ephy_testing_data.outputs.dataset_hash }}
+          restore-keys: ${{ runner.os }}-datasets-
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: neo-test-env-${{ matrix.python-version }}
+          python-version: "${{ matrix.python-version }}"
+
+      - name: Install testing dependencies
+        # testing environment is only created from yml if no cache was found
+        # restore-key hits should result in `cache-hit` == 'false'
+        #if: steps.cache-conda-env.outputs.cache-hit != 'true'
+        run: |
+          conda install pip numpy=${{ matrix.numpy-version }} -c conda-forge
+        # this command is for updating cache. We are resting removal.
+        # conda env update --name neo-test-env-${{ matrix.python-version }} --file environment_testing.yml --prune
+
+      - name: Install git-annex
+        # this is the trick from the spikeinterface repo for getting git-annex to work with datalad
+        # see https://github.com/SpikeInterface/spikeinterface/pull/3877 for more info
+        shell: bash
+        run: |
+          pip install datalad-installer
+          datalad-installer --sudo ok git-annex --method datalad/packages
+          git config --global filter.annex.process "git-annex filter-process"  # recommended for efficiency
+
+      - name: Configure git
+        run: |
+          git config --global user.email "neo_ci@fake_mail.com"
+          git config --global user.name "neo CI"
+
+      - name: Python version
+        run: |
+          which python
+          python --version
+
+      - name: Install neo including dependencies
+        # installation with dependencies is only required if no cache was found
+        # restore-key hits should result in `cache-hit` == 'false'
+        # if: steps.cache-conda-env.outputs.cache-hit != 'true'
+        run: |
+          pip install --upgrade -e .
+          pip install .[test]
+
+
+      - name: Install wine
+        run: |
+          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
+          sudo dpkg --add-architecture i386
+          sudo apt-get update -qq
+          sudo apt-get install -yqq --allow-downgrades libc6:i386 libgcc-s1:i386 libstdc++6:i386 wine
+
+      - name: Pip list
+        run: |
+          pip list
+
+      - name: Conda list
+        run: |
+          conda list
+
+      - name: Test with pytest
+        env:
+          HDF5_PLUGIN_PATH: ${{ github.workspace }}/hdf5_local_plugin_path
+          PLEXON2_TEST: true
+        run: |
+          # only neo.rawio and neo.io
+          pytest --cov=neo neo/test/rawiotest
+          pytest --cov=neo neo/test/iotest

doc/source/authors.rst

@@ -71,8 +71,8 @@ and may not be the current affiliation of a contributor.
 * Daniel P. Crepeau [30]
 * Divyansh Gupta [31]
 * Nate Dolensek [3]
-* Philipp Hornauer [32]
-* Robert Wolff
+* Philipp Hornauer [32, 45]
+* Robert Wolff [42]
 * Jules Lebert [33]
 * Benjamin Heasly
 * Baptiste Grimaud [34]
@@ -94,6 +94,7 @@ and may not be the current affiliation of a contributor.
 * Nina Kudryashova [37]
 * Rémi Proville [44]
 * Paul Adkisson [24]
+* Luiz Tauffer [24]
 
 1. Centre de Recherche en Neuroscience de Lyon, CNRS UMR5292 - INSERM U1028 - Université Claude Bernard Lyon 1
 2. Unité de Neuroscience, Information et Complexité, CNRS UPR 3293, Gif-sur-Yvette, France
@@ -136,9 +137,10 @@ and may not be the current affiliation of a contributor.
 39. Massachusetts General Hospital, Department of Molecular Biology
 40. Plexon Inc.
 41. Paris Brain Institute
-42. Istituto Italiano di Tecnologia (IIT), Italy
+42. Istituto Italiano di Tecnologia (IIT), Genoa, Italy
 43. University of Genoa, Italy
 44. AquiNeuro, SAS
+45. Maxwell Biosystems AG
 
 
 

doc/source/governance.rst

@@ -38,6 +38,7 @@ The current maintainers are:
 - Julia Sprenger (`@JuliaSprenger`_)
 - Michael Denker (`@mdenker`_)
 - Alessio Buccino (`@alejoe91`_)
+- Zach McKenzie (`@zm711`_)
 
 
 .. _`Neo maintainers team`: https://github.com/orgs/NeuralEnsemble/teams/neo-maintainers
@@ -47,3 +48,4 @@ The current maintainers are:
 .. _`@JuliaSprenger`: https://github.com/JuliaSprenger
 .. _`@mdenker`: https://github.com/mdenker
 .. _`@alejoe91`: https://github.com/alejoe91
+.. _`@zm711`: https://github.com/zm711

neo/core/spiketrainlist.py

@@ -113,8 +113,10 @@ def __init__(self, items=None, parent=None):
         self._channel_id_array = None
         self._all_channel_ids = None
         self._spiketrain_metadata = {}
-        if parent is not None and parent.__class__.__name__ != "Segment":
-            raise AttributeError("The parent class must be a Segment")
+
+        from .segment import Segment  # import here rather than at the top to avoid circular import
+        if parent is not None and not isinstance(parent, Segment):
+            raise AttributeError("If provided, the parent class must be a Segment")
         self.segment = parent
 
     @property

neo/rawio/blackrockrawio.py

@@ -597,10 +597,17 @@ def _parse_header(self):
             for c in range(spike_channels.size):
                 st_ann = seg_ann["spikes"][c]
                 channel_id, unit_id = self.internal_unit_ids[c]
-                unit_tag = {0: "unclassified", 255: "noise"}.get(unit_id, str(unit_id))
                 st_ann["channel_id"] = channel_id
                 st_ann["unit_id"] = unit_id
-                st_ann["unit_tag"] = unit_tag
+                if unit_id == 0:
+                    st_ann["unit_classification"] = "unclassified"
+                elif 1 <= unit_id <= 16:
+                    st_ann["unit_classification"] = "sorted"
+                elif unit_id == 255:
+                    st_ann["unit_classification"] = "noise"
+                else:  # 17-254 are reserved
+                    st_ann["unit_classification"] = "reserved"
+                st_ann["unit_tag"] = st_ann["unit_classification"]
                 st_ann["description"] = f"SpikeTrain channel_id: {channel_id}, unit_id: {unit_id}"
                 st_ann["file_origin"] = self._filenames["nev"] + ".nev"
 
@@ -1058,7 +1065,10 @@ def __read_nsx_data_variant_a(self, nsx_nb):
         filename = ".".join([self._filenames["nsx"], f"ns{nsx_nb}"])
 
         # get shape of data
-        shape = (int(self.__nsx_params["2.1"](nsx_nb)["nb_data_points"]), int(self.__nsx_basic_header[nsx_nb]["channel_count"]))
+        shape = (
+            int(self.__nsx_params["2.1"](nsx_nb)["nb_data_points"]),
+            int(self.__nsx_basic_header[nsx_nb]["channel_count"]),
+        )
         offset = int(self.__nsx_params["2.1"](nsx_nb)["bytes_in_headers"])
 
         # read nsx data
@@ -1251,7 +1261,19 @@ def __read_nev_data(self, nev_data_masks, nev_data_types):
         # read all raw data packets and markers
         dt0 = [("timestamp", ts_format), ("packet_id", "uint16"), ("value", f"S{data_size - header_skip}")]
 
-        raw_data = np.memmap(filename, offset=header_size, dtype=dt0, mode="r")
+        # expected number of data packets. We are not sure why, but it seems we can get partial data packets
+        # based on blackrock's own code this is okay so applying an int to round down is necessary to obtain the
+        # memory map of full packets and toss the partial packet.
+        # See reference: https://github.com/BlackrockNeurotech/Python-Utilities/blob/fa75aa671680306788e10d3d8dd625f9da4ea4f6/brpylib/brpylib.py#L580-L587
+        n_packets = int((self.__get_file_size(filename) - header_size) / data_size)
+
+        raw_data = np.memmap(
+            filename,
+            offset=header_size,
+            dtype=dt0,
+            shape=(n_packets,),
+            mode="r",
+        )
 
         masks = self.__nev_data_masks(raw_data["packet_id"])
         types = self.__nev_data_types(data_size)
@@ -1794,7 +1816,7 @@ def __nev_params(self, param_name):
                 hour=self.__nev_basic_header["hour"],
                 minute=self.__nev_basic_header["minute"],
                 second=self.__nev_basic_header["second"],
-                microsecond=self.__nev_basic_header["millisecond"],
+                microsecond=int(self.__nev_basic_header["millisecond"]) * 1000,
             ),
             "max_res": self.__nev_basic_header["timestamp_resolution"],
             "channel_ids": self.__nev_ext_header[b"NEUEVWAV"]["electrode_id"],

neo/rawio/intanrawio.py

@@ -35,7 +35,7 @@
     _signal_buffer_dtype,
     _spike_channel_dtype,
     _event_channel_dtype,
-) 
+)
 
 
 class IntanRawIO(BaseRawIO):

neo/rawio/maxwellrawio.py

@@ -242,18 +242,20 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, strea
             raise (e)
 
 
-_hdf_maxwell_error = """Maxwell file format is based on HDF5.
-The internal compression requires a custom plugin!!!
-This is a big pain for the end user.
-You, as a end user, should ask Maxwell company to change this.
-Please visit this page and install the missing decompression libraries:
-https://share.mxwbio.com/d/4742248b2e674a85be97/
-Then, link the decompression library by setting the `HDF5_PLUGIN_PATH` to your
-installation location, e.g. via
+_hdf_maxwell_error = """The MaxWell file compression requires a custom plugin.
+You can use the auto_install_maxwell_hdf5_compression_plugin() function or 
+(if it fails) install it manually:
+Download the missing decompression library:
+https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35
+Then, link the decompression library by setting the `HDF5_PLUGIN_PATH` to its location, 
+e.g. directly in Python via:
 os.environ['HDF5_PLUGIN_PATH'] = '/path/to/custom/hdf5/plugin/'
-
-Alternatively, you can use the auto_install_maxwell_hdf5_compression_plugin() below
-function that do it automagically.
+or in your shell via:
+export HDF5_PLUGIN_PATH=/path/to/custom/hdf5/plugin/
+You can also set the `HDF5_PLUGIN_PATH` environment variable in your shell
+configuration file (e.g. .bashrc, .bash_profile, .zshrc, etc.) to make it
+permanent. 
+See https://mxw.bio/MxW_Doc_Installing_Decompression_Library_to_load_MaxLab_Live_Recordings for more details.
 """
 
 
@@ -267,13 +269,17 @@ def auto_install_maxwell_hdf5_compression_plugin(hdf5_plugin_path=None, force_do
     hdf5_plugin_path.mkdir(exist_ok=True)
 
     if platform.system() == "Linux":
-        remote_lib = "https://share.mxwbio.com/d/4742248b2e674a85be97/files/?p=%2FLinux%2Flibcompression.so&dl=1"
+        remote_lib = "https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35/files/?p=%2FLinux%2Flibcompression.so&dl=1"
         local_lib = hdf5_plugin_path / "libcompression.so"
     elif platform.system() == "Darwin":
-        remote_lib = "https://share.mxwbio.com/d/4742248b2e674a85be97/files/?p=%2FMacOS%2Flibcompression.dylib&dl=1"
+        if platform.machine() == "arm64":
+            remote_lib = "https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35/files/?p=%2FMacOS%2FMac_arm64%2Flibcompression.dylib&dl=1"
+        else:
+            # Assuming x86_64 for MacOS
+            remote_lib = "https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35/files/?p=%2FMacOS%2FMac_x86_64%2Flibcompression.dylib&dl=1"
         local_lib = hdf5_plugin_path / "libcompression.dylib"
     elif platform.system() == "Windows":
-        remote_lib = "https://share.mxwbio.com/d/4742248b2e674a85be97/files/?p=%2FWindows%2Fcompression.dll&dl=1"
+        remote_lib = "https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35/files/?p=%2FWindows%2Fcompression.dll&dl=1"
         local_lib = hdf5_plugin_path / "compression.dll"
 
     if not force_download and local_lib.is_file():

neo/rawio/micromedrawio.py

@@ -51,7 +51,6 @@ def __init__(self, filename=""):
         self.filename = filename
 
     def _parse_header(self):
-
         with open(self.filename, "rb") as fid:
             f = StructFile(fid)
 
@@ -133,7 +132,11 @@ def _parse_header(self):
             sig_grounds = []
             for c in range(Num_Chan):
                 zname2, pos, length = zones["LABCOD"]
-                f.seek(pos + code[c] * 128 + 2, 0)
+                # Force code[c] which is currently a uint16 (or u2) into a int to prevent integer overflow
+                # for the following operation -- code[c] * 128 + 2.
+                # An integer overflow below may have side - effects including but not limited
+                # to having repeated channel names.
+                f.seek(pos + int(code[c]) * 128 + 2, 0)
 
                 chan_name = f.read(6).strip(b"\x00").decode("ascii")
                 ground = f.read(6).strip(b"\x00").decode("ascii")
@@ -274,7 +277,6 @@ def _event_count(self, block_index, seg_index, event_channel_index):
         return n
 
     def _get_event_timestamps(self, block_index, seg_index, event_channel_index, t_start, t_stop):
-
         raw_event = self._raw_events[event_channel_index][seg_index]
 
         # important : all events timing are related to the first segment t_start