Fix complicated corner cases in openephys legacy when small chunk when gap_mode=True

Author: samuelgarcia

neo/rawio/openephysrawio.py

@@ -87,7 +87,6 @@ def _parse_header(self):
             self._sigs_memmap[seg_index] = {}
             self._sig_has_gap[seg_index] = {}
 
-            all_sigs_length = []
             all_first_timestamps = []
             all_last_timestamps = []
             all_samplerate = []
@@ -107,26 +106,18 @@ def _parse_header(self):
                                       dtype=continuous_dtype, shape=(size, ))
                 self._sigs_memmap[seg_index][chan_index] = data_chan
 
-                # print(data_chan)
-                
-                # import matplotlib.pyplot as plt
-                # fig, ax = plt.subplots()
-                # ax.plot(data_chan['timestamp'])
-                # plt.show()
-
-                # all_sigs_length.append(data_chan.size * RECORD_SIZE)
                 all_first_timestamps.append(data_chan[0]['timestamp'])
                 all_last_timestamps.append(data_chan[-1]['timestamp'] + RECORD_SIZE)
                 all_samplerate.append(chan_info['sampleRate'])
 
                 # check for continuity (no gaps)
                 diff = np.diff(data_chan['timestamp'])
-                self._sig_has_gap[seg_index][chan_index] = not np.all(diff == RECORD_SIZE)
-                # if not np.all(diff == RECORD_SIZE) and not self._ignore_timestamps_errors:
-                #     raise ValueError(
-                #         'Not continuous timestamps for {}. ' \
-                #         'Maybe because recording was paused/stopped.'.format(continuous_filename)
-                #     )
+                channel_has_gaps = not np.all(diff == RECORD_SIZE)
+                self._sig_has_gap[seg_index][chan_index] = channel_has_gaps
+                
+                if channel_has_gaps:
+                    # protect against strange timestamp block like in file 'OpenEphys_SampleData_3' CH32
+                    assert np.median(diff) == RECORD_SIZE, f"This file has strange block timestamp for channel {chan_id}"
 
                 if seg_index == 0:
                     # add in channel list
@@ -339,17 +330,32 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop,
             # slow mode 
             for i, global_chan_index in enumerate(global_channel_indexes):
                 data = self._sigs_memmap[seg_index][global_chan_index]
-                t0 = data[0]['timestamp']
+                timestamp0 = data[0]['timestamp']
 
                 # find first block
-                block0 = np.searchsorted(data['timestamp'], t0 + i_start, side='right') - 1
-                shift0 = i_start + t0 - data[block0]['timestamp']
-                pos = RECORD_SIZE - shift0
-                sigs_chunk[:, i][:pos] = data[block0]['samples'][shift0:]
+                block0 = np.searchsorted(data['timestamp'], timestamp0 + i_start, side='right') - 1
+                block0_pos = data[block0]['timestamp'] - timestamp0
 
+                if i_start - block0_pos >  RECORD_SIZE:
+                    # the block has gap!!
+                    pos = - ((i_start - block0_pos) % RECORD_SIZE)
+                    block_index = block0 + 1
+                else:
+                    # the first block do not have gaps
+                    shift0 = i_start - block0_pos
+                    
+                    if shift0 + (i_stop - i_start) < RECORD_SIZE:
+                        # protect when only one small block
+                        pos = (i_stop - i_start)
+                        sigs_chunk[:, i][:pos] = data[block0]['samples'][shift0:shift0 + pos]
+                    else:
+                        
+                        pos = RECORD_SIZE - shift0
+                        sigs_chunk[:, i][:pos] = data[block0]['samples'][shift0:]
+                    block_index = block0 + 1
+
                 # full block
-                block_index = block0 + 1
-                while data[block_index]['timestamp'] - t0 < i_stop - RECORD_SIZE:
+                while block_index < data.size and data[block_index]['timestamp'] - timestamp0 < i_stop - RECORD_SIZE:
                     diff = data[block_index]['timestamp'] - data[block_index - 1]['timestamp']
                     if diff > RECORD_SIZE:
                         # gap detected need jump
@@ -361,7 +367,10 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop,
 
                 # last block
                 if pos < i_stop - i_start:
-                    sigs_chunk[:, i][pos:] = data[block_index]['samples'][:i_stop - i_start - pos]
+                    diff = data[block_index]['timestamp'] - data[block_index - 1]['timestamp']
+                    if diff == RECORD_SIZE:
+                        # ensure no gaps for last block
+                        sigs_chunk[:, i][pos:] = data[block_index]['samples'][:i_stop - i_start - pos]
 
         return sigs_chunk
 
@@ -505,6 +514,7 @@ def explore_folder(dirname):
     "100_CH0_2.continuous" ---> seg_index 1
     "100_CH0_N.continuous" ---> seg_index N-1
     """
+    print(dirname, type(dirname))
     filenames = os.listdir(dirname)
     filenames.sort()
 

neo/test/rawiotest/rawio_compliance.py

@@ -243,6 +243,7 @@ def read_analogsignals(reader):
                     chunks.append(raw_chunk)
                     i_start += chunksize
                 chunk_raw_sigs = np.concatenate(chunks, axis=0)
+
                 np.testing.assert_array_equal(ref_raw_sigs, chunk_raw_sigs)
 
 

neo/test/rawiotest/test_openephysrawio.py

@@ -10,24 +10,12 @@ class TestOpenEphysRawIO(BaseTestRawIO, unittest.TestCase, ):
         'openephys'
     ]
     entities_to_test = [
-        'openephys/OpenEphys_SampleData_1',
-        # 'OpenEphys_SampleData_2_(multiple_starts)',  # This not implemented this raise error
-        # 'OpenEphys_SampleData_3',
+        # 'openephys/OpenEphys_SampleData_1',
+        # this file has gaps and this is now handle corretly
+        'openephys/OpenEphys_SampleData_2_(multiple_starts)',
+        # 'openephys/OpenEphys_SampleData_3',
     ]
 
-    def test_raise_error_if_discontinuous_files(self):
-        # the case of discontinuous signals is NOT cover by the IO for the moment
-        # It must raise an error
-        reader = OpenEphysRawIO(dirname=self.get_local_path(
-            'openephys/OpenEphys_SampleData_2_(multiple_starts)'))
-        with self.assertRaises(ValueError):
-            reader.parse_header()
-        # if ignore_timestamps_errors=True, no exception is raised
-        reader = OpenEphysRawIO(dirname=self.get_local_path(
-            'openephys/OpenEphys_SampleData_2_(multiple_starts)'),
-                                ignore_timestamps_errors=True)
-        reader.parse_header()
-
 
     def test_raise_error_if_strange_timestamps(self):
         # In this dataset CH32 have strange timestamps