Merge branch 'master' into add_intan_stim

h-mayorquin · web-flow · commit 2c1583c79304 · 2025-04-01T13:13:48.000-06:00
diff --git a/neo/rawio/baserawio.py b/neo/rawio/baserawio.py
@@ -239,6 +239,184 @@ def __repr__(self):
 
         return txt
 
+    def _repr_html_(self):
+        """
+        HTML representation for the raw recording base.
+
+        Returns
+        -------
+        html : str
+            The HTML representation as a string.
+        """
+        html = []
+        html.append('<div style="font-family: Arial, sans-serif; max-width: 1000px; margin: 0 auto;">')
+
+        # Header
+        html.append(f'<h3 style="color: #2c3e50;">{self.__class__.__name__}: {self.source_name()}</h3>')
+
+        if self.is_header_parsed:
+            # Basic info
+            nb_block = self.block_count()
+            html.append(f"<p><strong>nb_block:</strong> {nb_block}</p>")
+            nb_seg = [self.segment_count(i) for i in range(nb_block)]
+            html.append(f"<p><strong>nb_segment:</strong> {nb_seg}</p>")
+
+            # CSS for tables - using only black, white, and gray colors
+            html.append(
+                """
+            <style>
+                #{unique_id} table.neo-table {{
+                    border-collapse: collapse;
+                    width: 100%;
+                    margin-bottom: 20px;
+                    font-size: 14px;
+                    color: inherit;
+                    background-color: transparent;
+                }}
+                #{unique_id} table.neo-table th,
+                #{unique_id} table.neo-table td {{
+                    border: 1px solid #888;
+                    padding: 8px;
+                    text-align: left;
+                }}
+                #{unique_id} table.neo-table th {{
+                    background-color: rgba(128,128,128,0.2);
+                }}
+                #{unique_id} table.neo-table tr:nth-child(even) {{
+                    background-color: rgba(128,128,128,0.1);
+                }}
+                #{unique_id} details {{
+                    margin-bottom: 15px;
+                    border: 1px solid rgba(128,128,128,0.3);
+                    border-radius: 4px;
+                    overflow: hidden;
+                    background-color: transparent;
+                }}
+                #{unique_id} summary {{
+                    padding: 10px;
+                    background-color: rgba(128,128,128,0.2);
+                    cursor: pointer;
+                    font-weight: bold;
+                    color: inherit;
+                }}
+                #{unique_id} details[open] summary {{
+                    border-bottom: 1px solid rgba(128,128,128,0.3);
+                }}
+                #{unique_id} .table-container {{
+                    padding: 10px;
+                    overflow-x: auto;
+                    background-color: transparent;
+                }}
+            </style>
+            """
+            )
+
+            # Signal Streams
+            signal_streams = self.header["signal_streams"]
+            if signal_streams.size > 0:
+                html.append("<details>")
+                html.append("<summary>Signal Streams</summary>")
+                html.append('<div class="table-container">')
+                html.append('<table class="neo-table">')
+                html.append("<thead><tr><th>Name</th><th>ID</th><th>Buffer ID</th><th>Channel Count</th></tr></thead>")
+                html.append("<tbody>")
+
+                for i, stream in enumerate(signal_streams):
+                    html.append("<tr>")
+                    html.append(f'<td>{stream["name"]}</td>')
+                    html.append(f'<td>{stream["id"]}</td>')
+                    html.append(f'<td>{stream["buffer_id"]}</td>')
+                    html.append(f"<td>{self.signal_channels_count(i)}</td>")
+                    html.append("</tr>")
+
+                html.append("</tbody></table>")
+                html.append("</div>")
+                html.append("</details>")
+
+            # Signal Channels
+            signal_channels = self.header["signal_channels"]
+            if signal_channels.size > 0:
+                html.append("<details>")
+                html.append("<summary>Signal Channels</summary>")
+                html.append('<div class="table-container">')
+                html.append('<table class="neo-table">')
+                html.append(
+                    "<thead><tr><th>Name</th><th>ID</th><th>Sampling Rate</th><th>Data Type</th><th>Units</th><th>Gain</th><th>Offset</th><th>Stream ID</th><th>Buffer ID</th></tr></thead>"
+                )
+                html.append("<tbody>")
+
+                for channel in signal_channels:
+                    html.append("<tr>")
+                    html.append(f'<td>{channel["name"]}</td>')
+                    html.append(f'<td>{channel["id"]}</td>')
+                    html.append(f'<td>{channel["sampling_rate"]}</td>')
+                    html.append(f'<td>{channel["dtype"]}</td>')
+                    html.append(f'<td>{channel["units"]}</td>')
+                    html.append(f'<td>{channel["gain"]}</td>')
+                    html.append(f'<td>{channel["offset"]}</td>')
+                    html.append(f'<td>{channel["stream_id"]}</td>')
+                    html.append(f'<td>{channel["buffer_id"]}</td>')
+                    html.append("</tr>")
+
+                html.append("</tbody></table>")
+                html.append("</div>")
+                html.append("</details>")
+
+            # Spike Channels
+            spike_channels = self.header["spike_channels"]
+            if spike_channels.size > 0:
+                html.append("<details>")
+                html.append("<summary>Spike Channels</summary>")
+                html.append('<div class="table-container">')
+                html.append('<table class="neo-table">')
+                html.append(
+                    "<thead><tr><th>Name</th><th>ID</th><th>WF Units</th><th>WF Gain</th><th>WF Offset</th><th>WF Left Sweep</th><th>WF Sampling Rate</th></tr></thead>"
+                )
+                html.append("<tbody>")
+
+                for channel in spike_channels:
+                    html.append("<tr>")
+                    html.append(f'<td>{channel["name"]}</td>')
+                    html.append(f'<td>{channel["id"]}</td>')
+                    html.append(f'<td>{channel["wf_units"]}</td>')
+                    html.append(f'<td>{channel["wf_gain"]}</td>')
+                    html.append(f'<td>{channel["wf_offset"]}</td>')
+                    html.append(f'<td>{channel["wf_left_sweep"]}</td>')
+                    html.append(f'<td>{channel["wf_sampling_rate"]}</td>')
+                    html.append("</tr>")
+
+                html.append("</tbody></table>")
+                html.append("</div>")
+                html.append("</details>")
+
+            # Event Channels
+            event_channels = self.header["event_channels"]
+            if event_channels.size > 0:
+                html.append("<details>")
+                html.append("<summary>Event Channels</summary>")
+                html.append('<div class="table-container">')
+                html.append('<table class="neo-table">')
+                html.append("<thead><tr><th>Name</th><th>ID</th><th>Type</th></tr></thead>")
+                html.append("<tbody>")
+
+                for channel in event_channels:
+                    html.append("<tr>")
+                    html.append(f'<td>{channel["name"]}</td>')
+                    html.append(f'<td>{channel["id"]}</td>')
+                    html.append(
+                        f'<td>{channel["type"].decode("utf-8") if isinstance(channel["type"], bytes) else channel["type"]}</td>'
+                    )
+                    html.append("</tr>")
+
+                html.append("</tbody></table>")
+                html.append("</div>")
+                html.append("</details>")
+        else:
+            html.append("<p><em>Call <code>parse_header()</code> to load the reader data.</p>")
+
+        html.append("</div>")
+        return "\n".join(html)
+
     def _generate_minimal_annotations(self):
         """
         Helper function that generates a nested dict for annotations.
diff --git a/neo/rawio/blackrockrawio.py b/neo/rawio/blackrockrawio.py
@@ -276,12 +276,19 @@ def _parse_header(self):
             self.internal_unit_ids = []  # pair of chan['packet_id'], spikes['unit_class_nb']
             for i in range(len(self.__nev_ext_header[b"NEUEVWAV"])):
 
-                channel_id = self.__nev_ext_header[b"NEUEVWAV"]["electrode_id"][i]
+                # electrode_id values are stored at uint16 which can overflow when
+                # multiplying by 1000 below. We convert to a regular python int which
+                # won't overflow
+                channel_id = int(self.__nev_ext_header[b"NEUEVWAV"]["electrode_id"][i])
 
                 chan_mask = spikes["packet_id"] == channel_id
                 chan_spikes = spikes[chan_mask]
+
+                # all `unit_class_nb` is uint8. Also will have issues with overflow
+                # cast this to python int
                 all_unit_id = np.unique(chan_spikes["unit_class_nb"])
                 for u, unit_id in enumerate(all_unit_id):
+                    unit_id = int(unit_id)
                     self.internal_unit_ids.append((channel_id, unit_id))
                     name = f"ch{channel_id}#{unit_id}"
                     _id = f"Unit {1000 * channel_id + unit_id}"
diff --git a/neo/rawio/spikegadgetsrawio.py b/neo/rawio/spikegadgetsrawio.py
@@ -81,7 +81,7 @@ def __init__(self, filename="", selected_streams=None):
     def _source_name(self):
         return self.filename
 
-    def _produce_ephys_channel_ids(self, n_total_channels, n_channels_per_chip):
+    def _produce_ephys_channel_ids(self, n_total_channels, n_channels_per_chip, missing_hw_chans):
         """Compute the channel ID labels for subset of spikegadgets recordings
         The ephys channels in the .rec file are stored in the following order:
         hwChan ID of channel 0 of first chip, hwChan ID of channel 0 of second chip, ..., hwChan ID of channel 0 of Nth chip,
@@ -94,14 +94,18 @@ def _produce_ephys_channel_ids(self, n_total_channels, n_channels_per_chip):
         This doesn't work for all types of spikegadgets
         see: https://github.com/NeuralEnsemble/python-neo/issues/1517
 
+        If there are any missing hardware channels, they must be specified in missing_hw_chans.
+        See: https://github.com/NeuralEnsemble/python-neo/issues/1592
         """
         ephys_channel_ids_list = []
-        for hw_channel in range(n_channels_per_chip):
-            hw_channel_list = [
-                hw_channel + chip * n_channels_per_chip for chip in range(int(n_total_channels / n_channels_per_chip))
-            ]
-            ephys_channel_ids_list.append(hw_channel_list)
-        return [channel for channel_list in ephys_channel_ids_list for channel in channel_list]
+        for local_hw_channel in range(n_channels_per_chip):
+            n_chips = int(n_total_channels / n_channels_per_chip)
+            for chip in range(n_chips):
+                global_hw_chan = local_hw_channel + chip * n_channels_per_chip
+                if global_hw_chan in missing_hw_chans:
+                    continue
+                ephys_channel_ids_list.append(local_hw_channel + chip * n_channels_per_chip)
+        return ephys_channel_ids_list
 
     def _parse_header(self):
         # parse file until "</Configuration>"
@@ -126,7 +130,8 @@ def _parse_header(self):
         sconf = root.find("SpikeConfiguration")
 
         self._sampling_rate = float(hconf.attrib["samplingRate"])
-        num_ephy_channels = int(hconf.attrib["numChannels"])
+        num_ephy_channels_xml = int(hconf.attrib["numChannels"])
+        num_ephy_channels = num_ephy_channels_xml
 
         # check for agreement with number of channels in xml
         sconf_channels = np.sum([len(x) for x in sconf])
@@ -220,7 +225,11 @@ def _parse_header(self):
             # we can only produce these channels for a subset of spikegadgets setup. If this criteria isn't
             # true then we should just use the raw_channel_ids and let the end user sort everything out
             if num_ephy_channels % num_chan_per_chip == 0:
-                channel_ids = self._produce_ephys_channel_ids(num_ephy_channels, num_chan_per_chip)
+                all_hw_chans = [int(schan.attrib["hwChan"]) for trode in sconf for schan in trode]
+                missing_hw_chans = set(range(num_ephy_channels)) - set(all_hw_chans)
+                channel_ids = self._produce_ephys_channel_ids(
+                    num_ephy_channels_xml, num_chan_per_chip, missing_hw_chans
+                )
                 raw_channel_ids = False
             else:
                 raw_channel_ids = True
diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py
@@ -376,28 +376,15 @@ def get_segment_tuple(info):
     for info in info_list:
         info["seg_index"] = segment_tuple_to_segment_index[get_segment_tuple(info)]
 
-    # Probe index calculation
-    # The calculation is ordered by slot, port, dock in that order, this is the number that appears in the filename
-    # after imec when using native names (e.g. imec0, imec1, etc.)
-    def get_probe_tuple(info):
-        slot = normalize(info.get("probe_slot"))
-        port = normalize(info.get("probe_port"))
-        dock = normalize(info.get("probe_dock"))
-        return (slot, port, dock)
-
-    # TODO: handle one box case
-    info_list_imec = [info for info in info_list if info.get("device") != "nidq"]
-    unique_probe_tuples = {get_probe_tuple(info) for info in info_list_imec}
-    sorted_probe_keys = sorted(unique_probe_tuples)
-    probe_tuple_to_probe_index = {key: idx for idx, key in enumerate(sorted_probe_keys)}
-
     for info in info_list:
-        if info.get("device") == "nidq":
-            info["device_index"] = ""  # TODO: Handle multi nidq case, maybe use meta["typeNiEnabled"]
+        # device_kind is imec, nidq
+        if info.get("device_kind") == "imec":
+            info["device_index"] = info["device"].split("imec")[-1]
         else:
-            info["device_index"] = probe_tuple_to_probe_index[get_probe_tuple(info)]
+            info["device_index"] = ""  # TODO: Handle multi nidq case, maybe use meta["typeNiEnabled"]
 
-    # Define stream base on device [imec|nidq], device_index and stream_kind [ap|lf] for imec
+    # Define stream base on device_kind [imec|nidq], device_index and stream_kind [ap|lf] for imec
+    # Stream format is "{device_kind}{device_index}.{stream_kind}"
     for info in info_list:
         device_kind = info["device_kind"]
         device_index = info["device_index"]
@@ -524,6 +511,7 @@ def extract_stream_info(meta_file, meta):
         # NIDQ case
         has_sync_trace = False
 
+    # This is the original name that the file had. It might not match the current name if the user changed it
     bin_file_path = meta["fileName"]
     fname = Path(bin_file_path).stem
 
diff --git a/neo/test/generate_datasets.py b/neo/test/generate_datasets.py
@@ -165,7 +165,7 @@ def random_segment():
         rec_datetime=random_datetime(),
         **random_annotations(4),
     )
-    
+
     n_sigs = random.randint(0, 5)
     for i in range(n_sigs):
         seg.analogsignals.append(random_signal())
diff --git a/neo/test/rawiotest/test_spikegadgetsrawio.py b/neo/test/rawiotest/test_spikegadgetsrawio.py
@@ -1,7 +1,9 @@
 import unittest
+from pathlib import Path
 
 from neo.rawio.spikegadgetsrawio import SpikeGadgetsRawIO
 from neo.test.rawiotest.common_rawio_test import BaseTestRawIO
+from numpy.testing import assert_array_equal
 
 
 class TestSpikeGadgetsRawIO(
@@ -16,6 +18,36 @@ class TestSpikeGadgetsRawIO(
         "spikegadgets/SpikeGadgets_test_data_2xNpix1.0_20240318_173658.rec",
     ]
 
+    def test_parse_header_missing_channels(self):
 
-if __name__ == "__main__":
-    unittest.main()
+        file_path = Path(self.get_local_path("spikegadgets/SL18_D19_S01_F01_BOX_SLP_20230503_112642_stubbed.rec"))
+        reader = SpikeGadgetsRawIO(filename=file_path)
+        reader.parse_header()
+
+        assert_array_equal(
+            reader.header["signal_channels"]["id"],
+            # fmt: off
+            [
+                'ECU_Ain1', 'ECU_Ain2', 'ECU_Ain3', 'ECU_Ain4', 'ECU_Ain5', 'ECU_Ain6',
+                'ECU_Ain7', 'ECU_Ain8', 'ECU_Aout1', 'ECU_Aout2', 'ECU_Aout3', 'ECU_Aout4', '0',
+                '32', '96', '160', '192', '224', '1', '33', '65', '97', '161', '193', '225', '2', '34',
+                '98', '162', '194', '226', '3', '35', '67', '99', '163', '195', '227', '4', '36',
+                '100', '164', '196', '228', '5', '37', '69', '101', '165', '197', '229', '6', '38',
+                '102', '166', '198', '230', '7', '39', '71', '103', '167', '199', '231', '8', '40',
+                '72', '104', '136', '168', '200', '232', '9', '41', '73', '105', '137', '169', '201',
+                '233', '10', '42', '74', '106', '138', '170', '202', '234', '11', '43', '75', '107',
+                '139', '171', '203', '235', '12', '44', '76', '108', '140', '172', '204', '236', '13',
+                '45', '77', '109', '141', '173', '205', '237', '14', '46', '78', '110', '142', '174',
+                '206', '238', '15', '47', '79', '111', '143', '175', '207', '239', '80', '144', '176',
+                '208', '240', '17', '49', '81', '145', '177', '209', '241', '82', '146', '178', '210',
+                '242', '19', '51', '83', '147', '179', '211', '243', '84', '148', '180', '212', '244',
+                '21', '53', '85', '149', '181', '213', '245', '86', '150', '182', '214', '246', '23',
+                '55', '87', '151', '183', '215', '247', '24', '56', '88', '152', '184', '216', '248',
+                '25', '57', '89', '121', '153', '185', '217', '249', '26', '58', '90', '154', '186',
+                '218', '250', '27', '59', '91', '123', '155', '187', '219', '251', '28', '60', '92',
+                '156', '188', '220', '252', '29', '61', '93', '125', '157', '189', '221', '253', '30',
+                '62', '94', '158', '190', '222', '254', '31', '63', '95', '127', '159', '191', '223',
+                '255'
+            ]
+            # fmt: on
+        )
diff --git a/neo/test/rawiotest/test_spikeglxrawio.py b/neo/test/rawiotest/test_spikeglxrawio.py
@@ -43,6 +43,19 @@ class TestSpikeGLXRawIO(BaseTestRawIO, unittest.TestCase):
         "spikeglx/multi_trigger_multi_gate/CatGT/Supercat-A",
     ]
 
+    def test_loading_only_one_probe_in_multi_probe_scenario(self):
+        from pathlib import Path
+        local_path_multi_probe_path = Path(self.get_local_path("spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0"))
+        gate_folder_path = local_path_multi_probe_path / "5-19-2022-CI0_g0"
+        probe_folder_path = gate_folder_path / "5-19-2022-CI0_g0_imec1"
+        
+        rawio = SpikeGLXRawIO(probe_folder_path)
+        rawio.parse_header()
+        
+        expected_stream_names = ["imec1.ap", "imec1.lf"]
+        actual_stream_names = rawio.header["signal_streams"]["name"].tolist()
+        assert actual_stream_names == expected_stream_names, f"Expected {expected_stream_names}, but got {actual_stream_names}" 
+
     def test_with_location(self):
         rawio = SpikeGLXRawIO(self.get_local_path("spikeglx/Noise4Sam_g0"), load_channel_location=True)
         rawio.parse_header()

Original file line number	Diff line number	Diff line change
`@@ -165,7 +165,7 @@ def random_segment():`
`165`	`165`	`rec_datetime=random_datetime(),`
`166`	`166`	`**random_annotations(4),`
`167`	`167`	`)`
`168`		`-`
	`168`	`+`
`169`	`169`	`n_sigs = random.randint(0, 5)`
`170`	`170`	`for i in range(n_sigs):`
`171`	`171`	`seg.analogsignals.append(random_signal())`