Fixing some errors related to missing times data

Author: juhuntenburg

brainbox/behavior/dlc.py

@@ -440,26 +440,37 @@ def plot_motion_energy_hist(camera_dict, trials_df):
               'body': '#035382'}
 
     start_window, end_window = plt_window(trials_df['stimOn_times'])
+    missing_data = []
     for cam in camera_dict.keys():
-        try:
-            motion_energy = zscore(camera_dict[cam]['motion_energy'], nan_policy='omit')
-            start_idx = insert_idx(camera_dict[cam]['times'], start_window)
-            end_idx = np.array(start_idx + int(WINDOW_LEN * SAMPLING[cam]), dtype='int64')
-            me_all = [motion_energy[start_idx[i]:end_idx[i]] for i in range(len(start_idx))]
-            times = np.arange(len(me_all[0])) / SAMPLING[cam] + WINDOW_LAG
-            me_mean = np.mean(me_all, axis=0)
-            me_std = np.std(me_all, axis=0) / np.sqrt(len(me_all))
-            plt.plot(times, me_mean, label=f'{cam} cam', color=colors[cam], linewidth=2)
-            plt.fill_between(times, me_mean + me_std, me_mean - me_std, color=colors[cam], alpha=0.2)
-        except AttributeError:
-            logger.warning(f"Cannot load motion energy AND times data for {cam} camera")
+        if (camera_dict[cam]['motion_energy'] is not None and len(camera_dict[cam]['motion_energy']) > 0
+                and camera_dict[cam]['times'] is not None and len(camera_dict[cam]['times']) > 0):
+            try:
+                motion_energy = zscore(camera_dict[cam]['motion_energy'], nan_policy='omit')
+                start_idx = insert_idx(camera_dict[cam]['times'], start_window)
+                end_idx = np.array(start_idx + int(WINDOW_LEN * SAMPLING[cam]), dtype='int64')
+                me_all = [motion_energy[start_idx[i]:end_idx[i]] for i in range(len(start_idx))]
+                times = np.arange(len(me_all[0])) / SAMPLING[cam] + WINDOW_LAG
+                me_mean = np.mean(me_all, axis=0)
+                me_std = np.std(me_all, axis=0) / np.sqrt(len(me_all))
+                plt.plot(times, me_mean, label=f'{cam} cam', color=colors[cam], linewidth=2)
+                plt.fill_between(times, me_mean + me_std, me_mean - me_std, color=colors[cam], alpha=0.2)
+            except AttributeError:
+                logger.warning(f"Cannot load motion energy and/or times data for {cam} camera")
+                missing_data.append(cam)
+        else:
+            logger.warning(f"Data missing or empty for motion energy and/or times data for {cam} camera")
+            missing_data.append(cam)
 
     plt.xticks([-0.5, 0, 0.5, 1, 1.5])
     plt.ylabel('z-scored motion energy [a.u.]')
     plt.xlabel('time [sec]')
     plt.axvline(x=0, label='stimOn', linestyle='--', c='k')
     plt.legend(loc='lower right')
     plt.title('Motion Energy')
+    if len(missing_data) > 0:
+        ax = plt.gca()
+        ax.text(.95, .35, f"Data incomplete for\n{' and '.join(missing_data)} camera", color='r', fontsize=10,
+                horizontalalignment='right', verticalalignment='center', transform=ax.transAxes)
     return plt.gca()
 
 

ibllib/pipes/ephys_preprocessing.py

@@ -903,15 +903,13 @@ def _run(self, overwrite=False, run_qc=True, plot_qc=True):
                     dlc = pd.read_parquet(dlc_file)
                     dlc_thresh = likelihood_threshold(dlc, 0.9)
                     # try to load respective camera times
-                    try:
-                        dlc_t = np.load(next(Path(self.session_path).joinpath('alf').glob(f'_ibl_{cam}Camera.times.*npy')))
-                        times = True
-                    except StopIteration:
-                        _logger.error(f'No camera.times found for {cam} camera. '
+                    dlc_t = np.load(next(Path(self.session_path).joinpath('alf').glob(f'_ibl_{cam}Camera.times.*npy')))
+                    times = True
+                    if dlc_t.shape[0] == 0:
+                        _logger.error(f'camera.times empty for {cam} camera. '
                                       f'Computations using camera.times will be skipped')
                         self.status = -1
                         times = False
-
                     # These features are only computed from left and right cam
                     if cam in ('left', 'right'):
                         features = pd.DataFrame()

ibllib/plots/figures.py

@@ -297,6 +297,11 @@ def dlc_qc_plot(eid, one=None):
             else:
                 logger.warning(f"Could not load _ibl_{cam}Camera.{feat} some DLC QC plots have to be skipped.")
                 data[f'{cam}_{feat}'] = None
+            # Sometimes there is a file but the object is empty
+            if len(data[f'{cam}_{feat}']) == 0:
+                logger.warning(f"Object loaded from _ibl_{cam}Camera.{feat} is empty, some plots have to be skipped.")
+                data[f'{cam}_{feat}'] = None
+                
     # Session data
     for alf_object in ['trials', 'wheel', 'licks']:
         try:
@@ -349,7 +354,7 @@ def dlc_qc_plot(eid, one=None):
         ax = plt.subplot(2, 5, i + 1)
         ax.text(-0.1, 1.15, ascii_uppercase[i], transform=ax.transAxes, fontsize=16, fontweight='bold')
         # Check if any of the inputs is None
-        if any([v is None for v in panel[1].values()]):
+        if any([v is None for v in panel[1].values()]) or any([v.values() is None for v in panel[1].values() if isinstance(v, dict)]):
             ax.text(.5, .5, f"Data incomplete\n{panel[0].__name__}", color='r', fontweight='bold',
                     fontsize=12, horizontalalignment='center', verticalalignment='center', transform=ax.transAxes)
             plt.axis('off')