Merge pull request #453 from int-brain-lab/postdlc_cams

Postdlc cams

Author: juhuntenburg

brainbox/behavior/dlc.py

@@ -458,16 +458,17 @@ def plot_motion_energy_hist(camera_dict, trials_df):
                 motion_energy = zscore(camera_dict[cam]['motion_energy'], nan_policy='omit')
                 try:
                     start_idx = insert_idx(camera_dict[cam]['times'], start_window)
+                    end_idx = np.array(start_idx + int(WINDOW_LEN * SAMPLING[cam]), dtype='int64')
+                    me_all = [motion_energy[start_idx[i]:end_idx[i]] for i in range(len(start_idx))]
+                    me_all = [m for m in me_all if len(m) > 0]
+                    times = np.arange(len(me_all[0])) / SAMPLING[cam] + WINDOW_LAG
+                    me_mean = np.mean(me_all, axis=0)
+                    me_std = np.std(me_all, axis=0) / np.sqrt(len(me_all))
+                    plt.plot(times, me_mean, label=f'{cam} cam', color=colors[cam], linewidth=2)
+                    plt.fill_between(times, me_mean + me_std, me_mean - me_std, color=colors[cam], alpha=0.2)
                 except ValueError:
-                    logger.error("Camera.times are outside of the trial windows")
-                    raise
-                end_idx = np.array(start_idx + int(WINDOW_LEN * SAMPLING[cam]), dtype='int64')
-                me_all = [motion_energy[start_idx[i]:end_idx[i]] for i in range(len(start_idx))]
-                times = np.arange(len(me_all[0])) / SAMPLING[cam] + WINDOW_LAG
-                me_mean = np.mean(me_all, axis=0)
-                me_std = np.std(me_all, axis=0) / np.sqrt(len(me_all))
-                plt.plot(times, me_mean, label=f'{cam} cam', color=colors[cam], linewidth=2)
-                plt.fill_between(times, me_mean + me_std, me_mean - me_std, color=colors[cam], alpha=0.2)
+                    logger.error(f"{cam}Camera camera.times are outside of the trial windows")
+                    missing_data.append(cam)
             except AttributeError:
                 logger.warning(f"Cannot load motion energy and/or times data for {cam} camera")
                 missing_data.append(cam)
@@ -484,7 +485,7 @@ def plot_motion_energy_hist(camera_dict, trials_df):
     if len(missing_data) > 0:
         ax = plt.gca()
         ax.text(.95, .35, f"Data incomplete for\n{' and '.join(missing_data)} camera", color='r', fontsize=10,
-                horizontalalignment='right', verticalalignment='center', transform=ax.transAxes)
+                fontweight='bold', horizontalalignment='right', verticalalignment='center', transform=ax.transAxes)
     return plt.gca()
 
 

ibllib/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.10.1"
+__version__ = "2.10.2"
 import warnings
 
 from ibllib.misc import logger_config

ibllib/pipes/ephys_preprocessing.py

@@ -1068,27 +1068,28 @@ class EphysPostDLC(tasks.Task):
                                  # the following are required for the DLC plot only
                                  # they are not strictly required, some plots just might be skipped
                                  # In particular the raw videos don't need to be downloaded as they can be streamed
-                                 ('_iblrig_bodyCamera.raw.mp4', 'raw_video_data', False),
-                                 ('_iblrig_leftCamera.raw.mp4', 'raw_video_data', False),
-                                 ('_iblrig_rightCamera.raw.mp4', 'raw_video_data', False),
-                                 ('rightROIMotionEnergy.position.npy', 'alf', False),
-                                 ('leftROIMotionEnergy.position.npy', 'alf', False),
-                                 ('bodyROIMotionEnergy.position.npy', 'alf', False),
-                                 ('_ibl_trials.choice.npy', 'alf', False),
-                                 ('_ibl_trials.feedbackType.npy', 'alf', False),
-                                 ('_ibl_trials.feedback_times.npy', 'alf', False),
-                                 ('_ibl_trials.stimOn_times.npy', 'alf', False),
-                                 ('_ibl_wheel.position.npy', 'alf', False),
-                                 ('_ibl_wheel.timestamps.npy', 'alf', False),
+                                 ('_iblrig_bodyCamera.raw.mp4', 'raw_video_data', True),
+                                 ('_iblrig_leftCamera.raw.mp4', 'raw_video_data', True),
+                                 ('_iblrig_rightCamera.raw.mp4', 'raw_video_data', True),
+                                 ('rightROIMotionEnergy.position.npy', 'alf', True),
+                                 ('leftROIMotionEnergy.position.npy', 'alf', True),
+                                 ('bodyROIMotionEnergy.position.npy', 'alf', True),
+                                 ('_ibl_trials.choice.npy', 'alf', True),
+                                 ('_ibl_trials.feedbackType.npy', 'alf', True),
+                                 ('_ibl_trials.feedback_times.npy', 'alf', True),
+                                 ('_ibl_trials.stimOn_times.npy', 'alf', True),
+                                 ('_ibl_wheel.position.npy', 'alf', True),
+                                 ('_ibl_wheel.timestamps.npy', 'alf', True),
                                  ],
                  # More files are required for all panels of the DLC QC plot to function
                  'output_files': [('_ibl_leftCamera.features.pqt', 'alf', True),
                                   ('_ibl_rightCamera.features.pqt', 'alf', True),
                                   ('licks.times.npy', 'alf', True),
-                                  ('dlc_qc_plot.png', 'snapshot', False)]
+                                  # ('dlc_qc_plot.png', 'snapshot', False)
+                                  ]
                  }
 
-    def _run(self, overwrite=False, run_qc=True, plot_qc=True):
+    def _run(self, overwrite=True, run_qc=True, plot_qc=True):
         """
         Run the EphysPostDLC task. Returns a list of file locations for the output files in signature. The created plot
         (dlc_qc_plot.png) is not returned, but saved in session_path/snapshots and uploaded to Alyx as a note.
@@ -1121,18 +1122,24 @@ def _run(self, overwrite=False, run_qc=True, plot_qc=True):
                     dlc = pd.read_parquet(dlc_file)
                     dlc_thresh = likelihood_threshold(dlc, 0.9)
                     # try to load respective camera times
-                    dlc_t = np.load(next(Path(self.session_path).joinpath('alf').glob(f'_ibl_{cam}Camera.times.*npy')))
-                    times = True
-                    if dlc_t.shape[0] == 0:
-                        _logger.error(f'camera.times empty for {cam} camera. '
-                                      f'Computations using camera.times will be skipped')
+                    try:
+                        dlc_t = np.load(next(Path(self.session_path).joinpath('alf').glob(f'_ibl_{cam}Camera.times.*npy')))
+                        times = True
+                        if dlc_t.shape[0] == 0:
+                            _logger.error(f'camera.times empty for {cam} camera. '
+                                          f'Computations using camera.times will be skipped')
+                            self.status = -1
+                            times = False
+                        elif dlc_t.shape[0] < len(dlc_thresh):
+                            _logger.error(f'Camera times shorter than DLC traces for {cam} camera. '
+                                          f'Computations using camera.times will be skipped')
+                            self.status = -1
+                            times = 'short'
+                    except StopIteration:
                         self.status = -1
                         times = False
-                    elif dlc_t.shape[0] < len(dlc_thresh):
-                        _logger.error(f'Camera times shorter than DLC traces for {cam} camera. '
+                        _logger.error(f'No camera.times for {cam} camera. '
                                       f'Computations using camera.times will be skipped')
-                        self.status = -1
-                        times = 'short'
                     # These features are only computed from left and right cam
                     if cam in ('left', 'right'):
                         features = pd.DataFrame()
@@ -1161,7 +1168,7 @@ def _run(self, overwrite=False, run_qc=True, plot_qc=True):
                         output_files.append(features_file)
 
                     # For all cams, compute DLC qc if times available
-                    if times is True or times == 'short' and run_qc:
+                    if run_qc is True and times in [True, 'short']:
                         # Setting download_data to False because at this point the data should be there
                         qc = DlcQC(self.session_path, side=cam, one=self.one, download_data=False)
                         qc.run(update=True)

release_notes.md

@@ -1,5 +1,8 @@
 ## Release Note 2.10
 
+## Release Note 2.10.2 2022-02-28
+- Small fixes to local server task queues
+
 ## Release Note 2.10.1 2022-02-22
 - Authenticate alyx user in Task class
 - Some fixes to make dlc_qc_plot in EphysPostDLC more reliable