allow for revisions in tasks

Author: mayofaulkner

ibllib/io/extractors/video_motion.py

@@ -111,10 +111,12 @@ def load_data(self, download=False):
             self.data.camera_times = {vidio.label_from_path(url): ts for ts, url in zip(cam.data, cam.url)}
         else:
             alf_path = self.session_path / 'alf'
-            self.data.wheel = alfio.load_object(alf_path, 'wheel', short_keys=True)
-            self.data.trials = alfio.load_object(alf_path, 'trials')
+            wheel_path = next(alf_path.rglob('*wheel.timestamps*')).parent
+            self.data.wheel = alfio.load_object(wheel_path, 'wheel', short_keys=True)
+            trials_path = next(alf_path.rglob('*trials.table*')).parent
+            self.data.trials = alfio.load_object(trials_path, 'trials')
             self.data.camera_times = {vidio.label_from_path(x): alfio.load_file_content(x) for x in
-                                      alf_path.glob('*Camera.times*')}
+                                      alf_path.rglob('*Camera.times*')}
         assert all(x is not None for x in self.data.values())
 
     def _set_eid_or_path(self, session_path_or_eid):
@@ -428,14 +430,15 @@ def fix_keys(alf_object):
             return ob
 
         alf_path = self.session_path.joinpath('alf')
-        wheel = (fix_keys(alfio.load_object(alf_path, 'wheel')) if location == 'SDSC' else alfio.load_object(alf_path, 'wheel'))
+        wheel_path = next(alf_path.rglob('*wheel.timestamps*')).parent
+        wheel = (fix_keys(alfio.load_object(wheel_path, 'wheel')) if location == 'SDSC' else alfio.load_object(wheel_path, 'wheel'))
         self.wheel_timestamps = wheel.timestamps
         # Compute interpolated wheel position and wheel times
         wheel_pos, self.wheel_time = wh.interpolate_position(wheel.timestamps, wheel.position, freq=1000)
         # Compute wheel velocity
         self.wheel_vel, _ = wh.velocity_filtered(wheel_pos, 1000)
         # Load in original camera times
-        self.camera_times = alfio.load_file_content(next(alf_path.glob(f'_ibl_{self.label}Camera.times*.npy')))
+        self.camera_times = alfio.load_file_content(next(alf_path.rglob(f'_ibl_{self.label}Camera.times*.npy')))
         self.camera_path = str(next(self.session_path.joinpath('raw_video_data').glob(f'_iblrig_{self.label}Camera.raw*.mp4')))
         self.camera_meta = vidio.get_video_meta(self.camera_path)
 
@@ -473,8 +476,8 @@ def fix_keys(alf_object):
         # We attempt to load in some behavior data (trials and dlc). This is only needed for the summary plots, having
         # trial aligned paw velocity (from the dlc) is a nice sanity check to make sure the alignment went well
         try:
-            self.trials = alfio.load_file_content(next(alf_path.glob('_ibl_trials.table*.pqt')))
-            self.dlc = alfio.load_file_content(next(alf_path.glob(f'_ibl_{self.label}Camera.dlc*.pqt')))
+            self.trials = alfio.load_file_content(next(alf_path.rglob('_ibl_trials.table*.pqt')))
+            self.dlc = alfio.load_file_content(next(alf_path.rglob(f'_ibl_{self.label}Camera.dlc*.pqt')))
             self.dlc = likelihood_threshold(self.dlc)
             self.behavior = True
         except (ALFObjectNotFound, StopIteration):

ibllib/pipes/tasks.py

@@ -387,6 +387,12 @@ def assert_expected(self, expected_files, silent=False):
         files = []
         for expected_file in expected_files:
             actual_files = list(Path(self.session_path).rglob(str(Path(*filter(None, reversed(expected_file[:2]))))))
+            # Account for revisions
+            if len(actual_files) == 0:
+                expected_revision = (expected_file[0], expected_file[1] + '/#*', expected_file[2])
+                actual_files = list(
+                    Path(self.session_path).rglob(str(Path(*filter(None, reversed(expected_revision[:2]))))))
+
             if len(actual_files) == 0 and expected_file[2]:
                 everything_is_fine = False
                 if not silent:

ibllib/pipes/video_tasks.py

@@ -506,7 +506,7 @@ def _run(self, overwrite=True, run_qc=True, plot_qc=True):
             if exist and overwrite:
                 _logger.warning('EphysPostDLC outputs exist and overwrite=True, overwriting existing outputs.')
             # Find all available DLC files
-            dlc_files = list(Path(self.session_path).joinpath('alf').glob('_ibl_*Camera.dlc.*'))
+            dlc_files = list(Path(self.session_path).joinpath('alf').rglob('_ibl_*Camera.dlc.*'))
             for dlc_file in dlc_files:
                 _logger.debug(dlc_file)
             output_files = []
@@ -521,7 +521,7 @@ def _run(self, overwrite=True, run_qc=True, plot_qc=True):
                     dlc_thresh = likelihood_threshold(dlc, 0.9)
                     # try to load respective camera times
                     try:
-                        dlc_t = np.load(next(Path(self.session_path).joinpath('alf').glob(f'_ibl_{cam}Camera.times.*npy')))
+                        dlc_t = np.load(next(Path(self.session_path).joinpath('alf').rglob(f'_ibl_{cam}Camera.times.*npy')))
                         times = True
                         if dlc_t.shape[0] == 0:
                             _logger.error(f'camera.times empty for {cam} camera. '

ibllib/plots/figures.py

@@ -783,19 +783,15 @@ def dlc_qc_plot(session_path, one=None, device_collection='raw_video_data',
     assert any(data[f'{cam}_times'] is not None for cam in cameras), "No camera times data could be loaded, aborting."
 
     # Load session level data
-    for alf_object in ['trials', 'wheel', 'licks']:
+    for alf_object, collection in zip(['trials', 'wheel', 'licks'], [trials_collection, trials_collection, 'alf']):
         try:
-            if alf_object == 'licks':
-                data[f'{alf_object}'] = alfio.load_object(session_path.joinpath('alf'),
-                                                          alf_object)  # load locally
-            else:
-                data[f'{alf_object}'] = alfio.load_object(session_path.joinpath(trials_collection), alf_object)  # load locally
+            data[f'{alf_object}'] = alfio.load_object(session_path.joinpath(collection), alf_object)  # load locally
             continue
         except ALFObjectNotFound:
             pass
         try:
             # then try from alyx
-            data[f'{alf_object}'] = one.load_object(one.path2eid(session_path), alf_object, collection=trials_collection)
+            data[f'{alf_object}'] = one.load_object(one.path2eid(session_path), alf_object, collection=collection)
         except ALFObjectNotFound:
             logger.warning(f"Could not load {alf_object} object, some plots have to be skipped.")
             data[f'{alf_object}'] = None

ibllib/qc/camera.py

@@ -1137,8 +1137,9 @@ def load_data(self, download_data: bool = None,
         alf_path = self.session_path / 'alf'
         try:
             assert not extract_times
+            cam_path = next(alf_path.rglob(f'*{self.label}Camera.times*')).parent
             self.data['timestamps'] = alfio.load_object(
-                alf_path, f'{self.label}Camera', short_keys=True)['times']
+                cam_path, f'{self.label}Camera', short_keys=True)['times']
         except AssertionError:  # Re-extract
             kwargs = dict(video_path=self.video_path, labels=self.label)
             if self.sync == 'bpod':
@@ -1154,8 +1155,8 @@ def load_data(self, download_data: bool = None,
         wheel_keys = ('timestamps', 'position')
         try:
             # glob in case wheel data are in sub-collections
-            alf_path = next(alf_path.rglob('*wheel.timestamps*')).parent
-            self.data['wheel'] = alfio.load_object(alf_path, 'wheel', short_keys=True)
+            wheel_path = next(alf_path.rglob('*wheel.timestamps*')).parent
+            self.data['wheel'] = alfio.load_object(wheel_path, 'wheel', short_keys=True)
         except ALFObjectNotFound:
             # Extract from raw data
             if self.sync != 'bpod':

ibllib/qc/dlc.py

@@ -93,9 +93,11 @@ def load_data(self, download_data: bool = None) -> None:
         alf_path = self.session_path / 'alf'
 
         # Load times
-        self.data['camera_times'] = alfio.load_object(alf_path, f'{self.side}Camera')['times']
+        cam_path = next(alf_path.rglob(f'*{self.side}Camera.times*')).parent
+        self.data['camera_times'] = alfio.load_object(cam_path, f'{self.side}Camera')['times']
         # Load dlc traces
-        dlc_df = alfio.load_object(alf_path, f'{self.side}Camera', namespace='ibl')['dlc']
+        dlc_path = next(alf_path.rglob(f'*{self.side}Camera.dlc*')).parent
+        dlc_df = alfio.load_object(dlc_path, f'{self.side}Camera', namespace='ibl')['dlc']
         targets = np.unique(['_'.join(col.split('_')[:-1]) for col in dlc_df.columns])
         # Set values to nan if likelihood is too low
         dlc_coords = {}
@@ -106,11 +108,13 @@ def load_data(self, download_data: bool = None) -> None:
         self.data['dlc_coords'] = dlc_coords
 
         # load stim on times
-        self.data['stimOn_times'] = alfio.load_object(alf_path, 'trials', namespace='ibl')['stimOn_times']
+        trial_path = next(alf_path.rglob(f'*trials.table*')).parent
+        self.data['stimOn_times'] = alfio.load_object(trial_path, 'trials', namespace='ibl')['stimOn_times']
 
         # load pupil diameters
         if self.side in ['left', 'right']:
-            features = alfio.load_object(alf_path, f'{self.side}Camera', namespace='ibl')['features']
+            feat_path = next(alf_path.rglob(f'*{self.side}Camera.features*')).parent
+            features = alfio.load_object(feat_path, f'{self.side}Camera', namespace='ibl')['features']
             self.data['pupilDiameter_raw'] = features['pupilDiameter_raw']
             self.data['pupilDiameter_smooth'] = features['pupilDiameter_smooth']