Merge branch 'release/2.10.3'

Author: juhuntenburg

brainbox/behavior/dlc.py

@@ -458,16 +458,17 @@ def plot_motion_energy_hist(camera_dict, trials_df):
                 motion_energy = zscore(camera_dict[cam]['motion_energy'], nan_policy='omit')
                 try:
                     start_idx = insert_idx(camera_dict[cam]['times'], start_window)
+                    end_idx = np.array(start_idx + int(WINDOW_LEN * SAMPLING[cam]), dtype='int64')
+                    me_all = [motion_energy[start_idx[i]:end_idx[i]] for i in range(len(start_idx))]
+                    me_all = [m for m in me_all if len(m) > 0]
+                    times = np.arange(len(me_all[0])) / SAMPLING[cam] + WINDOW_LAG
+                    me_mean = np.mean(me_all, axis=0)
+                    me_std = np.std(me_all, axis=0) / np.sqrt(len(me_all))
+                    plt.plot(times, me_mean, label=f'{cam} cam', color=colors[cam], linewidth=2)
+                    plt.fill_between(times, me_mean + me_std, me_mean - me_std, color=colors[cam], alpha=0.2)
                 except ValueError:
-                    logger.error("Camera.times are outside of the trial windows")
-                    raise
-                end_idx = np.array(start_idx + int(WINDOW_LEN * SAMPLING[cam]), dtype='int64')
-                me_all = [motion_energy[start_idx[i]:end_idx[i]] for i in range(len(start_idx))]
-                times = np.arange(len(me_all[0])) / SAMPLING[cam] + WINDOW_LAG
-                me_mean = np.mean(me_all, axis=0)
-                me_std = np.std(me_all, axis=0) / np.sqrt(len(me_all))
-                plt.plot(times, me_mean, label=f'{cam} cam', color=colors[cam], linewidth=2)
-                plt.fill_between(times, me_mean + me_std, me_mean - me_std, color=colors[cam], alpha=0.2)
+                    logger.error(f"{cam}Camera camera.times are outside of the trial windows")
+                    missing_data.append(cam)
             except AttributeError:
                 logger.warning(f"Cannot load motion energy and/or times data for {cam} camera")
                 missing_data.append(cam)
@@ -484,7 +485,7 @@ def plot_motion_energy_hist(camera_dict, trials_df):
     if len(missing_data) > 0:
         ax = plt.gca()
         ax.text(.95, .35, f"Data incomplete for\n{' and '.join(missing_data)} camera", color='r', fontsize=10,
-                horizontalalignment='right', verticalalignment='center', transform=ax.transAxes)
+                fontweight='bold', horizontalalignment='right', verticalalignment='center', transform=ax.transAxes)
     return plt.gca()
 
 

ibllib/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.10.2"
+__version__ = "2.10.3"
 import warnings
 
 from ibllib.misc import logger_config

ibllib/dsp/voltage.py

@@ -335,7 +335,7 @@ def decompress_destripe_cbin(sr_file, output_file=None, h=None, wrot=None, appen
     butter_kwargs, k_kwargs, spatial_fcn = _get_destripe_parameters(sr.fs, butter_kwargs, k_kwargs, k_filter)
     h = sr.geometry if h is None else h
     ncv = h['sample_shift'].size  # number of channels
-    output_file = sr.file_bin.with_suffix('.bin') if output_file is None else output_file
+    output_file = sr.file_bin.with_suffix('.bin') if output_file is None else Path(output_file)
     assert output_file != sr.file_bin
     taper = np.r_[0, scipy.signal.windows.cosine((SAMPLES_TAPER - 1) * 2), 0]
     # create the FFT stencils
@@ -481,8 +481,8 @@ def my_function(i_chunk, n_chunk):
         rms_data = np.frombuffer(rms_data, dtype=np.float32)
         assert(rms_data.shape[0] == time_data.shape[0] * ncv)
         rms_data = rms_data.reshape(time_data.shape[0], ncv)
-        np.save(Path(sr_file).parent.joinpath('_iblqc_ephysTimeRmsAP.rms.npy'), rms_data)
-        np.save(Path(sr_file).parent.joinpath('_iblqc_ephysTimeRmsAP.timestamps.npy'), time_data)
+        np.save(output_file.parent.joinpath('_iblqc_ephysTimeRmsAP.rms.npy'), rms_data)
+        np.save(output_file.parent.joinpath('_iblqc_ephysTimeRmsAP.timestamps.npy'), time_data)
 
 
 def rcoeff(x, y):

ibllib/pipes/ephys_preprocessing.py

@@ -1068,27 +1068,28 @@ class EphysPostDLC(tasks.Task):
                                  # the following are required for the DLC plot only
                                  # they are not strictly required, some plots just might be skipped
                                  # In particular the raw videos don't need to be downloaded as they can be streamed
-                                 ('_iblrig_bodyCamera.raw.mp4', 'raw_video_data', False),
-                                 ('_iblrig_leftCamera.raw.mp4', 'raw_video_data', False),
-                                 ('_iblrig_rightCamera.raw.mp4', 'raw_video_data', False),
-                                 ('rightROIMotionEnergy.position.npy', 'alf', False),
-                                 ('leftROIMotionEnergy.position.npy', 'alf', False),
-                                 ('bodyROIMotionEnergy.position.npy', 'alf', False),
-                                 ('_ibl_trials.choice.npy', 'alf', False),
-                                 ('_ibl_trials.feedbackType.npy', 'alf', False),
-                                 ('_ibl_trials.feedback_times.npy', 'alf', False),
-                                 ('_ibl_trials.stimOn_times.npy', 'alf', False),
-                                 ('_ibl_wheel.position.npy', 'alf', False),
-                                 ('_ibl_wheel.timestamps.npy', 'alf', False),
+                                 ('_iblrig_bodyCamera.raw.mp4', 'raw_video_data', True),
+                                 ('_iblrig_leftCamera.raw.mp4', 'raw_video_data', True),
+                                 ('_iblrig_rightCamera.raw.mp4', 'raw_video_data', True),
+                                 ('rightROIMotionEnergy.position.npy', 'alf', True),
+                                 ('leftROIMotionEnergy.position.npy', 'alf', True),
+                                 ('bodyROIMotionEnergy.position.npy', 'alf', True),
+                                 ('_ibl_trials.choice.npy', 'alf', True),
+                                 ('_ibl_trials.feedbackType.npy', 'alf', True),
+                                 ('_ibl_trials.feedback_times.npy', 'alf', True),
+                                 ('_ibl_trials.stimOn_times.npy', 'alf', True),
+                                 ('_ibl_wheel.position.npy', 'alf', True),
+                                 ('_ibl_wheel.timestamps.npy', 'alf', True),
                                  ],
                  # More files are required for all panels of the DLC QC plot to function
                  'output_files': [('_ibl_leftCamera.features.pqt', 'alf', True),
                                   ('_ibl_rightCamera.features.pqt', 'alf', True),
                                   ('licks.times.npy', 'alf', True),
-                                  ('dlc_qc_plot.png', 'snapshot', False)]
+                                  # ('dlc_qc_plot.png', 'snapshot', False)
+                                  ]
                  }
 
-    def _run(self, overwrite=False, run_qc=True, plot_qc=True):
+    def _run(self, overwrite=True, run_qc=True, plot_qc=True):
         """
         Run the EphysPostDLC task. Returns a list of file locations for the output files in signature. The created plot
         (dlc_qc_plot.png) is not returned, but saved in session_path/snapshots and uploaded to Alyx as a note.
@@ -1121,18 +1122,24 @@ def _run(self, overwrite=False, run_qc=True, plot_qc=True):
                     dlc = pd.read_parquet(dlc_file)
                     dlc_thresh = likelihood_threshold(dlc, 0.9)
                     # try to load respective camera times
-                    dlc_t = np.load(next(Path(self.session_path).joinpath('alf').glob(f'_ibl_{cam}Camera.times.*npy')))
-                    times = True
-                    if dlc_t.shape[0] == 0:
-                        _logger.error(f'camera.times empty for {cam} camera. '
-                                      f'Computations using camera.times will be skipped')
+                    try:
+                        dlc_t = np.load(next(Path(self.session_path).joinpath('alf').glob(f'_ibl_{cam}Camera.times.*npy')))
+                        times = True
+                        if dlc_t.shape[0] == 0:
+                            _logger.error(f'camera.times empty for {cam} camera. '
+                                          f'Computations using camera.times will be skipped')
+                            self.status = -1
+                            times = False
+                        elif dlc_t.shape[0] < len(dlc_thresh):
+                            _logger.error(f'Camera times shorter than DLC traces for {cam} camera. '
+                                          f'Computations using camera.times will be skipped')
+                            self.status = -1
+                            times = 'short'
+                    except StopIteration:
                         self.status = -1
                         times = False
-                    elif dlc_t.shape[0] < len(dlc_thresh):
-                        _logger.error(f'Camera times shorter than DLC traces for {cam} camera. '
+                        _logger.error(f'No camera.times for {cam} camera. '
                                       f'Computations using camera.times will be skipped')
-                        self.status = -1
-                        times = 'short'
                     # These features are only computed from left and right cam
                     if cam in ('left', 'right'):
                         features = pd.DataFrame()
@@ -1161,7 +1168,7 @@ def _run(self, overwrite=False, run_qc=True, plot_qc=True):
                         output_files.append(features_file)
 
                     # For all cams, compute DLC qc if times available
-                    if times is True or times == 'short' and run_qc:
+                    if run_qc is True and times in [True, 'short']:
                         # Setting download_data to False because at this point the data should be there
                         qc = DlcQC(self.session_path, side=cam, one=self.one, download_data=False)
                         qc.run(update=True)

release_notes.md

@@ -1,5 +1,9 @@
 ## Release Note 2.10
 
+## Release Note 2.10.3 2022-03-09
+- Fixes to EphysPostDLC
+- Small change to storing in dsp.voltage.decompress_destripe_cbin function
+
 ## Release Note 2.10.2 2022-02-28
 - Small fixes to local server task queues