[Neuralynx] adjust test for streams and precalculate signal length

Author: sprenger

neo/rawio/neuralynxrawio/ncssections.py

@@ -228,9 +228,9 @@ def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq):
             raise IOError("Sampling frequency in first record doesn't agree with header.")
         chanNum = ncsMemMap['channel_id'][0]
 
-        nb = NcsSections()
-        nb.sampFreqUsed = actualSampFreq
-        nb.microsPerSampUsed = NcsSectionsFactory.get_micros_per_samp_for_freq(actualSampFreq)
+        secs = NcsSections()
+        secs.sampFreqUsed = actualSampFreq
+        secs.microsPerSampUsed = NcsSectionsFactory.get_micros_per_samp_for_freq(actualSampFreq)
 
         # check if file is one block of records, which is often the case, and avoid full parse
         lastBlkI = ncsMemMap.shape[0] - 1
@@ -248,15 +248,15 @@ def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq):
             n_samples = NcsSection._RECORD_SIZE * lastBlkI
             curBlock = NcsSection(0, ts0, lastBlkI, lastBlkEndTime, n_samples)
 
-            nb.sects.append(curBlock)
-            return nb
+            secs.sects.append(curBlock)
+            return secs
 
         # otherwise need to scan looking for breaks
         else:
             blkOnePredTime = NcsSectionsFactory.calc_sample_time(actualSampFreq, ts0, nb0)
             curBlock = NcsSection(0, ts0, -1, -1, -1)
-            nb.sects.append(curBlock)
-            return NcsSectionsFactory._parseGivenActualFrequency(ncsMemMap, nb, chanNum, reqFreq,
+            secs.sects.append(curBlock)
+            return NcsSectionsFactory._parseGivenActualFrequency(ncsMemMap, secs, chanNum, reqFreq,
                                                                  blkOnePredTime)
 
     @staticmethod

neo/rawio/neuralynxrawio/neuralynxrawio.py

@@ -117,9 +117,6 @@ def _source_name(self):
         else:
             return self.dirname
 
-    # from memory_profiler import profile
-    #
-    # @profile()
     def _parse_header(self):
 
         stream_channels = []
@@ -192,7 +189,7 @@ def _parse_header(self):
             for idx, chan_id in enumerate(chan_ids):
                 chan_name = chan_names[idx]
 
-                chan_uid = (chan_name, chan_id)
+                chan_uid = (chan_name, str(chan_id))
                 if ext == 'ncs':
                     if info['sampling_rate'] not in ncs_sampling_rates:
                         ncs_sampling_rates.append(info['sampling_rate'])
@@ -310,6 +307,14 @@ def _parse_header(self):
             self._sigs_t_start = ncsSegTimestampLimits.t_start.copy()
             self._sigs_t_stop = ncsSegTimestampLimits.t_stop.copy()
 
+        # precompute signal lengths within segments
+        self._sigs_length = []
+        if self._sigs_memmaps:
+            for seg_idx, sig_container in enumerate(self._sigs_memmaps):
+                self._sigs_length.append({})
+                for chan_uid, sig_infos in sig_container.items():
+                    self._sigs_length[seg_idx][chan_uid] = int(sig_infos['nb_valid'].sum())
+
         # Determine timestamp limits in nev, nse file by scanning them.
         ts0, ts1 = None, None
         for _data_memmap in (self._spike_memmap, self._nev_memmap):
@@ -438,12 +443,12 @@ def _segment_t_stop(self, block_index, seg_index):
 
     def _get_signal_size(self, block_index, seg_index, stream_index):
         stream_id = self.header['signal_streams'][stream_index]['id']
-        channel_indexes = np.where(self.header['signal_channels']['stream_id'] == stream_id)[0]
+        stream_mask = self.header['signal_channels']['stream_id'] == stream_id
+        signals = self.header['signal_channels'][stream_mask]
 
-        if len(channel_indexes):
-            channel_index = channel_indexes[0]
-            chan = self.header['signal_channels'][channel_index]
-            return self._sigs_memmaps[seg_index][(chan['name'], int(chan['id']))]
+        if len(signals):
+            sig = signals[0]
+            return self._sigs_length[seg_index][(sig['name'], sig['id'])]
         else:
             return None
 
@@ -475,7 +480,8 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop,
         if i_start is None:
             i_start = 0
         if i_stop is None:
-            i_stop = self._sigs_length[seg_index]
+            i_stop = self.get_signal_size(block_index=block_index, seg_index=seg_index,
+                                          stream_index=stream_index)
 
         block_start = i_start // NcsSection._RECORD_SIZE
         block_stop = i_stop // NcsSection._RECORD_SIZE + 1
@@ -489,7 +495,7 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop,
         stream_mask = self.header['signal_channels']['stream_id'] == stream_id
 
         # channel_streams = self.
-        channel_ids = self.header['signal_channels'][stream_mask][channel_indexes]['id'].astype(int)
+        channel_ids = self.header['signal_channels'][stream_mask][channel_indexes]['id']
         channel_names = self.header['signal_channels'][stream_mask][channel_indexes]['name']
 
         # create buffer for samples
@@ -632,10 +638,9 @@ def scan_ncs_files(self, ncs_filenames):
             nlxHeader = NlxHeader(ncs_filename)
 
             if not chanSectMap or (chanSectMap and
-                    not NcsSectionsFactory._verifySectionsStructure(data,
-                    lastNcsSections)):
-                lastNcsSections = NcsSectionsFactory.build_for_ncs_file(data, nlxHeader)
-            chanSectMap[chan_uid] = [lastNcsSections, nlxHeader, ncs_filename]
+                    not NcsSectionsFactory._verifySectionsStructure(data, chan_ncs_sections)):
+                chan_ncs_sections = NcsSectionsFactory.build_for_ncs_file(data, nlxHeader)
+            chanSectMap[chan_uid] = [chan_ncs_sections, nlxHeader, ncs_filename]
             del data
 
         # Construct an inverse dictionary from NcsSections to list of associated chan_uids
@@ -646,7 +651,7 @@ def scan_ncs_files(self, ncs_filenames):
                 latest_sections = v[0]
             # time tolerance of +- one data package (in microsec)
             tolerance = 512 / min(v[0].sampFreqUsed, latest_sections.sampFreqUsed) *1e6
-            if v[0].is_equivalent(latest_sections, abs_tol=tolerance):
+            if not v[0].is_equivalent(latest_sections, abs_tol=tolerance):
                 revSectMap.setdefault(latest_sections, []).append(k)
             else:
                 revSectMap[v[0]] = [k]
@@ -658,7 +663,7 @@ def scan_ncs_files(self, ncs_filenames):
             raise IOError(f'ncs files have {len(revSectMap)} different sections '
                           f'structures. Unsupported configuration.')
 
-        seg_time_limits = SegmentTimeLimits(nb_segment=len(lastNcsSections.sects),
+        seg_time_limits = SegmentTimeLimits(nb_segment=len(chan_ncs_sections.sects),
                                             t_start=[], t_stop=[], length=[],
                                             timestamp_limits=[])
         memmaps = [{} for seg_index in range(seg_time_limits.nb_segment)]

neo/test/iotest/test_neuralynxio.py

@@ -275,7 +275,7 @@ def test_ncs(self):
 
             # check that data agrees in first segment first channel only
             for anasig_id, anasig in enumerate(block.segments[0].analogsignals):
-                chid = int(anasig.array_annotations['channel_ids'][0])
+                chid = anasig.array_annotations['channel_ids'][0]
 
                 chname = str(anasig.array_annotations['channel_names'][0])
                 chuid = (chname, chid)

neo/test/rawiotest/test_neuralynxrawio.py

@@ -39,7 +39,7 @@ def test_scan_ncs_files(self):
         # test values here from direct inspection of .ncs files
         self.assertEqual(rawio._nb_segment, 1)
         self.assertListEqual(rawio._timestamp_limits, [(0, 192000)])
-        self.assertEqual(rawio._sigs_length[0], 4608)
+        self.assertEqual(rawio._sigs_length[0][('unknown', '1')], 4608)
         self.assertEqual(rawio._sigs_t_start[0], 0)
         self.assertEqual(rawio._sigs_t_stop[0], 0.192)
         self.assertEqual(len(rawio._sigs_memmaps), 1)
@@ -51,8 +51,9 @@ def test_scan_ncs_files(self):
         rawio.parse_header()
         # test values here from direct inspection of .ncs files
         self.assertEqual(rawio._nb_segment, 1)
+        self.assertEqual(rawio.signal_streams_count(), 1)
         self.assertListEqual(rawio._timestamp_limits, [(266982936, 267162136)])
-        self.assertEqual(rawio._sigs_length[0], 5120)
+        self.assertEqual(rawio._sigs_length[0][('unknown', '13')], 5120)
         self.assertEqual(rawio._sigs_t_start[0], 266.982936)
         self.assertEqual(rawio._sigs_t_stop[0], 267.162136)
         self.assertEqual(len(rawio._sigs_memmaps), 1)
@@ -63,9 +64,13 @@ def test_scan_ncs_files(self):
         rawio.parse_header()
         # test values here from direct inspection of .ncs files
         self.assertEqual(rawio._nb_segment, 2)
+        self.assertEqual(rawio.signal_streams_count(), 1)
         self.assertListEqual(rawio._timestamp_limits, [(26122557633, 26162525633),
                                                        (26366360633, 26379704633)])
-        self.assertListEqual(rawio._sigs_length, [1278976, 427008])
+        self.assertDictEqual(rawio._sigs_length[0],
+                             {('Tet3a', '8'): 1278976, ('Tet3b', '9'): 1278976})
+        self.assertDictEqual(rawio._sigs_length[1],
+                             {('Tet3a', '8'): 427008, ('Tet3b', '9'): 427008})
         self.assertListEqual(rawio._sigs_t_stop, [26162.525633, 26379.704633])
         self.assertListEqual(rawio._sigs_t_start, [26122.557633, 26366.360633])
         self.assertEqual(len(rawio._sigs_memmaps), 2)  # check only that there are 2 memmaps
@@ -77,10 +82,13 @@ def test_scan_ncs_files(self):
         # test values here from direct inspection of .ncs file, except for 3rd block
         # t_stop, which is extended due to events past the last block of ncs records.
         self.assertEqual(rawio._nb_segment, 3)
+        self.assertEqual(rawio.signal_streams_count(), 1)
         self.assertListEqual(rawio._timestamp_limits, [(8408806811, 8427831990),
                                                        (8427832053, 8487768498),
                                                        (8487768561, 8515816549)])
-        self.assertListEqual(rawio._sigs_length, [608806, 1917967, 897536])
+        self.assertDictEqual(rawio._sigs_length[0], {('CSC1', '48'): 608806})
+        self.assertDictEqual(rawio._sigs_length[1], {('CSC1', '48'): 1917967})
+        self.assertDictEqual(rawio._sigs_length[2], {('CSC1', '48'): 897536})
         self.assertListEqual(rawio._sigs_t_stop, [8427.831990, 8487.768498, 8515.816549])
         self.assertListEqual(rawio._sigs_t_start, [8408.806811, 8427.832053, 8487.768561])
         self.assertEqual(len(rawio._sigs_memmaps), 3)  # check only that there are 3 memmaps