Merge branch 'master' into neuronexus

Author: zm711

codemeta.json

@@ -4,15 +4,15 @@
   "license": "https://spdx.org/licenses/BSD-3-Clause",
   "codeRepository": "https://github.com/NeuralEnsemble/python-neo",
   "contIntegration": "https://github.com/NeuralEnsemble/python-neo/actions",
-  "dateModified": "2024-08-01",
-  "downloadUrl": "https://files.pythonhosted.org/packages/0f/16/4e22eb38621183d56acde0abbe591f15e79c6332e9ec360fc5db171b39ab/neo-0.13.2.tar.gz",
+  "dateModified": "2024-08-28",
+  "downloadUrl": "https://files.pythonhosted.org/packages/08/4b/c863c6bff783e94c92cb814f6ae821b35e6463c5a66e809b6864d0c66b4e/neo-0.13.3.tar.gz",
   "issueTracker": "https://github.com/NeuralEnsemble/python-neo/issues",
   "name": "Neo",
-  "version": "0.13.2",
+  "version": "0.13.3",
   "identifier": "RRID:SCR_000634",
   "description": "Neo is a Python package for working with electrophysiology data in Python, together with support for reading a wide range of neurophysiology file formats, including Spike2, NeuroExplorer, AlphaOmega, Axon, Blackrock, Plexon, Tdt, and support for writing to a subset of these formats plus non-proprietary formats including HDF5.\n\nThe goal of Neo is to improve interoperability between Python tools for analyzing, visualizing and generating electrophysiology data by providing a common, shared object model. In order to be as lightweight a dependency as possible, Neo is deliberately limited to represention of data, with no functions for data analysis or visualization.\n\nNeo is used by a number of other software tools, including SpykeViewer (data analysis and visualization), Elephant (data analysis), the G-node suite (databasing), PyNN (simulations), tridesclous_ (spike sorting) and ephyviewer (data visualization).\n\nNeo implements a hierarchical data model well adapted to intracellular and extracellular electrophysiology and EEG data with support for multi-electrodes (for example tetrodes). Neo's data objects build on the quantities package, which in turn builds on NumPy by adding support for physical dimensions. Thus Neo objects behave just like normal NumPy arrays, but with additional metadata, checks for dimensional consistency and automatic unit conversion.",
   "applicationCategory": "neuroscience",
-  "releaseNotes": "https://neo.readthedocs.io/en/stable/releases/0.13.2.html",
+  "releaseNotes": "https://neo.readthedocs.io/en/stable/releases/0.13.3.html",
   "funding": "https://cordis.europa.eu/project/id/945539",
   "developmentStatus": "active",
   "referencePublication": "https://doi.org/10.3389/fninf.2014.00010",

doc/source/releases.rst

@@ -6,6 +6,7 @@ Release notes
 .. toctree::
    :maxdepth: 1
 
+   releases/0.13.3.rst
    releases/0.13.2.rst
    releases/0.13.1.rst
    releases/0.13.0.rst

doc/source/releases/0.13.3.rst

@@ -0,0 +1,33 @@
+========================
+Neo 0.13.3 release notes
+========================
+
+28 August 2024
+
+This release of Neo contains bug fixes, still with a focus on the planned 1.0 release,
+and will be the last release not to support NumPy 2.0.
+
+See all `pull requests`_ included in this release and the `list of closed issues`_.
+
+
+Updated dependencies
+--------------------
+
+Neo has a limit of NumPy >= 1.19.5, < 2.0.0 and Quantities >= 14.0.1, < 0.16.0
+
+
+Bug fixes and improvements in IO modules
+----------------------------------------
+
+Bug fixes and/or improvements have been made to :class:`PlexonIO`, :class:`SpikeGLXIO`,
+:class:`BiocamIO`.
+
+Acknowledgements
+----------------
+
+Thanks to Zach McKenzie, Heberto Mayorquin, and Alessio Buccino for their contributions to this release.
+
+
+.. _`pull requests`: https://github.com/NeuralEnsemble/python-neo/pulls?q=is%3Apr+is%3Aclosed+milestone%3A0.13.3
+
+.. _`list of closed issues`: https://github.com/NeuralEnsemble/python-neo/issues?q=is%3Aissue+is%3Aclosed+milestone%3A0.13.3

neo/rawio/biocamrawio.py

@@ -17,6 +17,7 @@
     _spike_channel_dtype,
     _event_channel_dtype,
 )
+from neo.core import NeoReadWriteError
 
 
 class BiocamRawIO(BaseRawIO):
@@ -122,15 +123,53 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, strea
             i_start = 0
         if i_stop is None:
             i_stop = self._num_frames
-        if channel_indexes is None:
-            channel_indexes = slice(None)
+
+        # read functions are different based on the version of biocam
         data = self._read_function(self._filehandle, i_start, i_stop, self._num_channels)
-        return data[:, channel_indexes]
 
+        # older style data returns array of (n_samples, n_channels), should be a view
+        # but if memory issues come up we should doublecheck out how the file is being stored
+        if data.ndim > 1:
+            if channel_indexes is None:
+                channel_indexes = slice(None)
+            sig_chunk = data[:, channel_indexes]
+
+        # newer style data returns an initial flat array (n_samples * n_channels)
+        # we iterate through channels rather than slicing
+        # Due to the fact that Neo and SpikeInterface tend to prefer slices we need to add
+        # some careful checks around slicing of None in the case we need to iterate through
+        # channels. First check if None. Then check if slice and only if slice check that it is slice(None)
+        else:
+            if channel_indexes is None:
+                channel_indexes = [ch for ch in range(self._num_channels)]
+            elif isinstance(channel_indexes, slice):
+                start = channel_indexes.start or 0
+                stop = channel_indexes.stop or self._num_channels
+                step = channel_indexes.step or 1
+                channel_indexes = [ch for ch in range(start, stop, step)]
+
+            sig_chunk = np.zeros((i_stop - i_start, len(channel_indexes)), dtype=data.dtype)
+            # iterate through channels to prevent loading all channels into memory which can cause
+            # memory exhaustion. See https://github.com/SpikeInterface/spikeinterface/issues/3303
+            for index, channel_index in enumerate(channel_indexes):
+                sig_chunk[:, index] = data[channel_index :: self._num_channels]
+
+        return sig_chunk
 
-def open_biocam_file_header(filename):
+
+def open_biocam_file_header(filename) -> dict:
     """Open a Biocam hdf5 file, read and return the recording info, pick the correct method to access raw data,
-    and return this to the caller."""
+    and return this to the caller
+
+    Parameters
+    ----------
+    filename: str
+        The file to be parsed
+
+    Returns
+    -------
+    dict
+        The information necessary to read a biocam file (gain, n_samples, n_channels, etc)."""
     import h5py
 
     rf = h5py.File(filename, "r")
@@ -154,9 +193,9 @@ def open_biocam_file_header(filename):
         elif file_format in (101, 102) or file_format is None:
             num_channels = int(rf["3BData/Raw"].shape[0] / num_frames)
         else:
-            raise Exception("Unknown data file format.")
+            raise NeoReadWriteError("Unknown data file format.")
 
-        # # get channels
+        # get channels
         channels = rf["3BRecInfo/3BMeaStreams/Raw/Chs"][:]
 
         # determine correct function to read data
@@ -166,14 +205,14 @@ def open_biocam_file_header(filename):
             elif signal_inv == -1:
                 read_function = readHDF5t_100_i
             else:
-                raise Exception("Unknown signal inversion")
+                raise NeoReadWriteError("Unknown signal inversion")
         else:
             if signal_inv == 1:
                 read_function = readHDF5t_101
             elif signal_inv == -1:
                 read_function = readHDF5t_101_i
             else:
-                raise Exception("Unknown signal inversion")
+                raise NeoReadWriteError("Unknown signal inversion")
 
         gain = (max_uv - min_uv) / (2**bit_depth)
         offset = min_uv
@@ -200,19 +239,22 @@ def open_biocam_file_header(filename):
         scale_factor = experiment_settings["ValueConverter"]["ScaleFactor"]
         sampling_rate = experiment_settings["TimeConverter"]["FrameRate"]
 
+        num_channels = None
         for key in rf:
             if key[:5] == "Well_":
                 num_channels = len(rf[key]["StoredChIdxs"])
                 if len(rf[key]["Raw"]) % num_channels:
-                    raise RuntimeError(f"Length of raw data array is not multiple of channel number in {key}")
+                    raise NeoReadWriteError(f"Length of raw data array is not multiple of channel number in {key}")
                 num_frames = len(rf[key]["Raw"]) // num_channels
                 break
-        try:
+
+        if num_channels is not None:
             num_channels_x = num_channels_y = int(np.sqrt(num_channels))
-        except NameError:
-            raise RuntimeError("No Well found in the file")
+        else:
+            raise NeoReadWriteError("No Well found in the file")
+
         if num_channels_x * num_channels_y != num_channels:
-            raise RuntimeError(f"Cannot determine structure of the MEA plate with {num_channels} channels")
+            raise NeoReadWriteError(f"Cannot determine structure of the MEA plate with {num_channels} channels")
         channels = 1 + np.concatenate(np.transpose(np.meshgrid(range(num_channels_x), range(num_channels_y))))
 
         gain = scale_factor * (max_uv - min_uv) / (max_digital - min_digital)
@@ -231,6 +273,11 @@ def open_biocam_file_header(filename):
         )
 
 
+######################################################################
+# Helper functions to obtain the raw data split by Biocam version.
+
+
+# return the full array for the old datasets
 def readHDF5t_100(rf, t0, t1, nch):
     return rf["3BData/Raw"][t0:t1]
 
@@ -239,15 +286,16 @@ def readHDF5t_100_i(rf, t0, t1, nch):
     return 4096 - rf["3BData/Raw"][t0:t1]
 
 
+# return flat array that we will iterate through
 def readHDF5t_101(rf, t0, t1, nch):
-    return rf["3BData/Raw"][nch * t0 : nch * t1].reshape((t1 - t0, nch), order="C")
+    return rf["3BData/Raw"][nch * t0 : nch * t1]
 
 
 def readHDF5t_101_i(rf, t0, t1, nch):
-    return 4096 - rf["3BData/Raw"][nch * t0 : nch * t1].reshape((t1 - t0, nch), order="C")
+    return 4096 - rf["3BData/Raw"][nch * t0 : nch * t1]
 
 
 def readHDF5t_brw4(rf, t0, t1, nch):
     for key in rf:
         if key[:5] == "Well_":
-            return rf[key]["Raw"][nch * t0 : nch * t1].reshape((t1 - t0, nch), order="C")
+            return rf[key]["Raw"][nch * t0 : nch * t1]

neo/rawio/blackrockrawio.py

@@ -986,7 +986,8 @@ def __read_nsx_dataheader_variant_c(
         index = 0
 
         if offset is None:
-            offset = self.__nsx_basic_header[nsx_nb]["bytes_in_headers"]
+            # This is read as an uint32 numpy scalar from the header so we transform it to python int
+            offset = int(self.__nsx_basic_header[nsx_nb]["bytes_in_headers"])
 
         ptp_dt = [
             ("reserved", "uint8"),

neo/rawio/medrawio.py

@@ -240,9 +240,18 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, strea
             self.sess.set_channel_active(self._stream_info[stream_index]["raw_chans"])
             num_channels = len(self._stream_info[stream_index]["raw_chans"])
             self.sess.set_reference_channel(self._stream_info[stream_index]["raw_chans"][0])
+
+        # in the case we have a slice or we give an ArrayLike we need to iterate through the channels
+        # in order to activate them.
         else:
-            if any(channel_indexes < 0):
-                raise IndexError(f"Can not index negative channels: {channel_indexes}")
+            if isinstance(channel_indexes, slice):
+                start = channel_indexes.start or 0
+                stop = channel_indexes.stop or len(self._stream_info[stream_index]["raw_chans"])
+                step = channel_indexes.step or 1
+                channel_indexes = [ch for ch in range(start, stop, step)]
+            else:
+                if any(channel_indexes < 0):
+                    raise IndexError(f"Can not index negative channels: {channel_indexes}")
             # Set all channels to be inactive, then selectively set some of them to be active
             self.sess.set_channel_inactive("all")
             for i, channel_idx in enumerate(channel_indexes):