Merge pull request #1786 from h-mayorquin/fix_neuralynx_streams

Separate signals on neuralynx and improve streams names

Author: h-mayorquin

neo/rawio/neuralynxrawio/neuralynxrawio.py

@@ -63,6 +63,9 @@
 from neo.rawio.neuralynxrawio.ncssections import NcsSection, NcsSectionsFactory
 from neo.rawio.neuralynxrawio.nlxheader import NlxHeader
 
+# Named tuple for stream identification keys
+StreamKey = namedtuple('StreamKey', ['sampling_rate', 'input_range', 'filter_params'])
+
 
 class NeuralynxRawIO(BaseRawIO):
     """
@@ -134,6 +137,20 @@ class NeuralynxRawIO(BaseRawIO):
         ("samples", "int16", NcsSection._RECORD_SIZE),
     ]
 
+    # Filter parameter keys used for stream differentiation
+    _filter_keys = [
+        "DSPLowCutFilterEnabled",
+        "DspLowCutFrequency",
+        "DspLowCutFilterType",
+        "DspLowCutNumTaps",
+        "DSPHighCutFilterEnabled",
+        "DspHighCutFrequency",
+        "DspHighCutFilterType",
+        "DspHighCutNumTaps",
+        "DspDelayCompensation",
+        "DspFilterDelay_µs",
+    ]
+
     def __init__(
         self,
         dirname="",
@@ -268,26 +285,49 @@ def _parse_header(self):
 
                 chan_uid = (chan_name, str(chan_id))
                 if ext == "ncs":
-                    file_mmap = self._get_file_map(filename)
-                    n_packets = copy.copy(file_mmap.shape[0])
-                    if n_packets:
-                        t_start = copy.copy(file_mmap[0][0])
-                    else:  # empty file
-                        t_start = 0
-                    stream_prop = (float(info["sampling_rate"]), int(n_packets), float(t_start))
-                    if stream_prop not in stream_props:
-                        stream_props[stream_prop] = {"stream_id": len(stream_props), "filenames": [filename]}
+                    # Calculate gain for this channel
+                    gain = info["bit_to_microVolt"][idx]
+                    if info.get("input_inverted", False):
+                        gain *= -1
+
+                    # Build stream key from acquisition parameters
+                    sampling_rate = float(info["sampling_rate"])
+
+                    # Get InputRange for this specific channel
+                    # Normalized by NlxHeader to always be a list
+                    input_range = info.get("InputRange")
+                    if isinstance(input_range, list):
+                        input_range = input_range[idx] if idx < len(input_range) else input_range[0]
+
+                    # Build filter parameters tuple
+                    filter_params = []
+                    for key in self._filter_keys:
+                        if key in info:
+                            filter_params.append((key, info[key]))
+
+                    # Create stream key (channels with same key go in same stream)
+                    stream_key = StreamKey(
+                        sampling_rate=sampling_rate,
+                        input_range=input_range,
+                        filter_params=tuple(sorted(filter_params)),
+                    )
+
+                    if stream_key not in stream_props:
+                        stream_props[stream_key] = {
+                            "stream_id": len(stream_props),
+                            "filenames": [filename],
+                            "channels": set(),
+                        }
                     else:
-                        stream_props[stream_prop]["filenames"].append(filename)
-                    stream_id = stream_props[stream_prop]["stream_id"]
+                        stream_props[stream_key]["filenames"].append(filename)
+
+                    stream_id = stream_props[stream_key]["stream_id"]
+                    stream_props[stream_key]["channels"].add((chan_name, str(chan_id)))
                     # @zach @ramon : we need to discuss this split by channel buffer
                     buffer_id = ""
 
                     # a sampled signal channel
                     units = "uV"
-                    gain = info["bit_to_microVolt"][idx]
-                    if info.get("input_inverted", False):
-                        gain *= -1
                     offset = 0.0
                     signal_channels.append(
                         (
@@ -392,14 +432,40 @@ def _parse_header(self):
         event_channels = np.array(event_channels, dtype=_event_channel_dtype)
 
         if signal_channels.size > 0:
-            # ordering streams according from high to low sampling rates
-            stream_props = {k: stream_props[k] for k in sorted(stream_props, reverse=True)}
-            stream_names = [f"Stream (rate,#packet,t0): {sp}" for sp in stream_props]
-            stream_ids = [stream_prop["stream_id"] for stream_prop in stream_props.values()]
-            buffer_ids = ["" for sp in stream_props]
+            # Build DSP filter configuration registry: filter_id -> filter parameters dict
+            # Extract unique filter configurations from stream keys
+            unique_filter_tuples = {stream_key.filter_params for stream_key in stream_props.keys()}
+            _dsp_filter_configurations = {i: dict(filt) for i, filt in enumerate(sorted(unique_filter_tuples))}
+
+            # Create reverse mapping for looking up filter IDs during stream name construction
+            seen_filters = {filt: i for i, filt in enumerate(sorted(unique_filter_tuples))}
+
+            # Store DSP filter configurations as private instance attribute
+            # Keeping private for now - may expose via annotations or public API in future
+            self._dsp_filter_configurations = _dsp_filter_configurations
+
+            # Order streams by sampling rate (high to low)
+            # This is to keep some semblance of stability
+            ordered_stream_keys = sorted(stream_props.keys(), reverse=True, key=lambda x: x.sampling_rate)
+
+            stream_names = []
+            stream_ids = []
+            buffer_ids = []
+
+            for stream_id, stream_key in enumerate(ordered_stream_keys):
+                # Format stream name using namedtuple fields
+                dsp_filter_id = seen_filters[stream_key.filter_params]
+                voltage_mv = int(stream_key.input_range / 1000) if stream_key.input_range is not None else 0
+                stream_name = f"stream{stream_id}_{int(stream_key.sampling_rate)}Hz_{voltage_mv}mVRange_DSPFilter{dsp_filter_id}"
+
+                stream_names.append(stream_name)
+                stream_ids.append(str(stream_id))
+                buffer_ids.append("")
+
             signal_streams = list(zip(stream_names, stream_ids, buffer_ids))
         else:
             signal_streams = []
+            self._filter_configurations = {}
         signal_buffers = np.array([], dtype=_signal_buffer_dtype)
         signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)
 

neo/rawio/neuralynxrawio/nlxheader.py

@@ -199,6 +199,9 @@ def __init__(self, filename, props_only=False):
         if not props_only:
             self._setTimeDate(txt_header)
 
+        # Normalize all types to proper Python types
+        self._normalize_types()
+
     @staticmethod
     def get_text_header(filename):
         """
@@ -349,6 +352,70 @@ def _setTimeDate(self, txt_header):
             dt2 = sr.groupdict()
             self["recording_closed"] = dateutil.parser.parse(f"{dt2['date']} {dt2['time']}")
 
+    def _normalize_types(self):
+        """
+        Convert all header values to proper Python types.
+
+        This ensures that:
+        - Boolean strings ('True', 'False', 'Enabled', 'Disabled') become Python bools
+        - Numeric strings ('0.1', '8000') become Python floats/ints
+        - Single-element lists are extracted to scalars (for single-channel files)
+
+        This normalization makes the header values directly usable for
+        stream identification without additional conversion in NeuralynxRawIO.
+        """
+
+        # Convert boolean strings to actual booleans
+        bool_keys = [
+            'DSPLowCutFilterEnabled',
+            'DSPHighCutFilterEnabled',
+            'DspDelayCompensation',
+        ]
+
+        for key in bool_keys:
+            if key in self and isinstance(self[key], str):
+                if self[key] in ('True', 'Enabled'):
+                    self[key] = True
+                elif self[key] in ('False', 'Disabled'):
+                    self[key] = False
+
+        # Convert numeric strings to numbers
+        numeric_keys = [
+            'DspLowCutFrequency',
+            'DspHighCutFrequency',
+            'DspLowCutNumTaps',
+            'DspHighCutNumTaps',
+        ]
+
+        for key in numeric_keys:
+            if key in self and isinstance(self[key], str):
+                try:
+                    # Try int first
+                    if '.' not in self[key]:
+                        self[key] = int(self[key])
+                    else:
+                        self[key] = float(self[key])
+                except ValueError:
+                    # Keep as string if conversion fails
+                    pass
+
+        # Handle DspFilterDelay_µs (could be string or already converted)
+        delay_key = 'DspFilterDelay_µs'
+        if delay_key in self and isinstance(self[delay_key], str):
+            try:
+                self[delay_key] = int(self[delay_key])
+            except ValueError:
+                pass
+
+        # Extract single-channel InputRange from list
+        # For multi-channel files, keep as list
+        # For single-channel files, extract the single value
+        if 'InputRange' in self and isinstance(self['InputRange'], list):
+            if len(self['InputRange']) == 1:
+                # Single channel file: extract the value
+                self['InputRange'] = self['InputRange'][0]
+            # else: multi-channel, keep as list
+
     def type_of_recording(self):
         """
         Determines type of recording in Ncs file with this header.

neo/test/rawiotest/test_neuralynxrawio.py

@@ -213,6 +213,48 @@ def test_directory_in_data_folder(self):
             self.assertEqual(len(rawio.header["spike_channels"]), 8)
             self.assertEqual(len(rawio.header["event_channels"]), 2)
 
+    def test_two_streams_different_header_encoding(self):
+        """
+        Test that streams are correctly differentiated based on filter parameters.
+        This dataset contains eye-tracking and ephys channels with different filter settings.
+        """
+        from pathlib import Path
+
+        # Get the path using the same machinery as other tests
+        dname = self.get_local_path("neuralynx/two_streams_different_header_encoding")
+
+        # Test with Path object (as shown in user's notebook)
+        rawio = NeuralynxRawIO(dirname=Path(dname))
+        rawio.parse_header()
+
+        # Should have 2 streams due to different filter configurations
+        self.assertEqual(rawio.signal_streams_count(), 2)
+
+        # Check stream names follow the new naming convention
+        stream_names = [rawio.header["signal_streams"][i][0] for i in range(rawio.signal_streams_count())]
+
+        # Stream names should include sampling rate (Hz), voltage range (mV), and DSP filter ID
+        for stream_name in stream_names:
+            self.assertRegex(stream_name, r"stream\d+_\d+Hz_\d+mVRange_DSPFilter\d+")
+
+        # Verify we have the expected streams:
+        # - Eye-tracking channels (CSC145, CSC146): 32000Hz, 100mV range, low-cut disabled
+        # - Ephys channel (CSC76): 32000Hz, 1mV range, low-cut enabled
+        expected_names = {"stream0_32000Hz_100mVRange_DSPFilter0", "stream1_32000Hz_1mVRange_DSPFilter1"}
+        self.assertEqual(set(stream_names), expected_names)
+
+        # Verify DSP filter configurations are stored (private for now)
+        self.assertTrue(hasattr(rawio, "_dsp_filter_configurations"))
+        self.assertEqual(len(rawio._dsp_filter_configurations), 2)
+
+        # Verify filter 0 (eye-tracking): low-cut disabled
+        filter_0 = rawio._dsp_filter_configurations[0]
+        self.assertFalse(filter_0.get("DSPLowCutFilterEnabled", True))
+
+        # Verify filter 1 (ephys): low-cut enabled
+        filter_1 = rawio._dsp_filter_configurations[1]
+        self.assertTrue(filter_1.get("DSPLowCutFilterEnabled", False))
+
 
 class TestNcsRecordingType(BaseTestRawIO, unittest.TestCase):
     """