Merge branch 'release/2.1.2'

Author: juhuntenburg

ibllib/ephys/ephysqc.py

@@ -42,9 +42,10 @@ class EphysQC(base.QC):
     default database if not given.
     """
 
-    def __init__(self, probe_id, **kwargs):
+    def __init__(self, probe_id, session_path=None, **kwargs):
         super().__init__(probe_id, endpoint='insertions', **kwargs)
         self.pid = probe_id
+        self.session_path = session_path
         self.stream = kwargs.pop('stream', True)
         keys = ('ap', 'ap_meta', 'lf', 'lf_meta')
         self.data = Bunch.fromkeys(keys)
@@ -57,7 +58,9 @@ def _ensure_required_data(self):
         """
         assert self.one is not None, 'ONE instance is required to ensure required data'
         eid, pname = self.one.pid2eid(self.pid)
-        self.probe_path = self.one.eid2path(eid).joinpath('raw_ephys_data', pname)
+        if self.session_path is None:
+            self.session_path = self.one.eid2path(eid)
+        self.probe_path = Path(self.session_path).joinpath('raw_ephys_data', pname)
         # Check if there is at least one meta file available
         meta_files = list(self.probe_path.rglob('*.meta'))
         assert len(meta_files) != 0, f'No meta files in {self.probe_path}'

ibllib/io/extractors/base.py

@@ -220,7 +220,14 @@ def get_pipeline(session_path):
 
 
 def _get_pipeline_from_task_type(stype):
+    """
+    Returns the pipeline from the task type. Some tasks types directly define the pipeline
+    :param stype: session_type or task extractor type
+    :return:
+    """
     if 'ephys' in stype:
         return 'ephys'
     elif stype in ['habituation', 'training', 'biased', 'biased_opto']:
         return 'training'
+    else:
+        return stype

ibllib/pipes/ephys_preprocessing.py

@@ -69,7 +69,7 @@ def _run(self, overwrite=False):
             pids = [p['id'] for p in create_alyx_probe_insertions(self.session_path, one=self.one)]
         qc_files = []
         for pid in pids:
-            eqc = ephysqc.EphysQC(pid, one=self.one)
+            eqc = ephysqc.EphysQC(pid, session_path=self.session_path, one=self.one)
             qc_files.extend(eqc.run(update=True, overwrite=overwrite))
         return qc_files
 

ibllib/qc/dlc.py

@@ -14,6 +14,7 @@
 
 from ibllib.qc import base
 import one.alf.io as alfio
+from one.alf.exceptions import ALFObjectNotFound
 from one.alf.spec import is_session_path
 from iblutil.util import Bunch
 
@@ -23,7 +24,6 @@
 class DlcQC(base.QC):
     """A class for computing camera QC metrics"""
 
-    dstypes = ['camera.dlc', 'camera.times']
     bbox = {
         'body': {
             'xrange': range(201, 500),
@@ -42,16 +42,17 @@ class DlcQC(base.QC):
     def __init__(self, session_path_or_eid, side, **kwargs):
         """
         :param session_path_or_eid: A session eid or path
+        :param side: The camera to run QC on
         :param log: A logging.Logger instance, if None the 'ibllib' logger is used
         :param one: An ONE instance for fetching and setting the QC on Alyx
-        :param camera: The camera to run QC on, if None QC is run for all three cameras.
         """
+        # Make sure the type of camera is chosen
+        self.side = side
         # When an eid is provided, we will download the required data by default (if necessary)
         download_data = not is_session_path(session_path_or_eid)
         self.download_data = kwargs.pop('download_data', download_data)
         super().__init__(session_path_or_eid, **kwargs)
         self.data = Bunch()
-        self.side = side
 
         # QC outcomes map
         self.metrics = None
@@ -95,15 +96,18 @@ def _ensure_required_data(self):
         it an exception is raised.
         :return:
         """
-        assert self.one is not None, 'ONE required to download data'
-        for dstype in self.dstypes:
-            dataset = self.one.type2datasets(self.eid, dstype, details=True)
-            present = (
-                self.one._download_datasets(dataset)
-                if self.download_data
-                else (next(self.session_path.rglob(d), None) for d in dataset['rel_path'])
-            )
-            assert (not dataset.empty and all(present)), f'Dataset {dstype} not found'
+        # Check if data available locally
+        for ds in [f'_ibl_{self.side}Camera.dlc.*', f'_ibl_{self.side}Camera.times.*']:
+            if not next(self.session_path.rglob(ds), None):
+                # If download is allowed, try to download
+                if self.download_data is True:
+                    assert self.one is not None, 'ONE required to download data'
+                    try:
+                        self.one.load_dataset(self.eid, ds, download_only=True)
+                    except ALFObjectNotFound:
+                        raise AssertionError(f'Dataset {ds} not found locally and failed to download')
+                else:
+                    raise AssertionError(f'Dataset {ds} not found locally and download_data is False')
 
     def run(self, update: bool = False, **kwargs) -> (str, dict):
         """

ibllib/tests/qc/test_dlc_qc.py

@@ -33,12 +33,19 @@ def tearDownClass(cls) -> None:
     def test_ensure_data(self):
         self.qc.eid = self.eid
         self.qc.download_data = False
-        # If data for this session exists locally, overwrite the methods so it is not found
-        if self.one.eid2path(self.eid).exists():
-            self.qc.one.to_eid = lambda _: self.eid
-            self.qc.one._download_datasets = lambda _: None
-        with self.assertRaises(AssertionError):
+        # Remove file so that the test fails as intended
+        if self.qc.session_path.exists():
+            self.qc.session_path.joinpath('alf/_ibl_leftCamera.dlc.pqt').unlink()
+        with self.assertRaises(AssertionError) as excp:
             self.qc.run(update=False)
+        msg = excp.exception.args[0]
+        self.assertEqual(msg, 'Dataset _ibl_leftCamera.dlc.* not found locally and download_data is False')
+        # Set download_data to True. Data is not in the database so we expect a (different) error trying to download
+        self.qc.download_data = True
+        with self.assertRaises(AssertionError) as excp:
+            self.qc.run(update=False)
+        msg = excp.exception.args[0]
+        self.assertEqual(msg, 'Dataset _ibl_leftCamera.dlc.* not found locally and failed to download')
 
     def test_check_time_trace_length_match(self):
         self.qc.data['dlc_coords'] = {'nose_tip': np.ones((2, 20)), 'pupil_r': np.ones((2, 20))}

release_notes.md

@@ -3,9 +3,12 @@
 - destriping as pykilosort internal pre-processing 
 - NP2 probe framework for splitting shanks and LFP band
 - Extension of task module to rerun from different locations
-### Release Notes 2.1.0 2021-10-06
+### Release Notes 2.1.1 2021-10-06
 - RawEphysQC tasks computes median RMS from samples for .ap data (stored in _iblqc_ephysChannels.RMS)
 - New EphysQC class
+### Release Notes 2.1.2 2021-10-14
+- Fix issue with RawEphysQC that was not looking in local Subjects folder for data
+- Fix ensure_required_data in DlcQc
 
 ## Release Notes 2.0
 ### Release Notes 2.0.1 2021-08-07

setup.py

@@ -24,7 +24,7 @@
 
 setup(
     name='ibllib',
-    version='2.1.1',
+    version='2.1.2',
     python_requires='>={}.{}'.format(*REQUIRED_PYTHON),
     description='IBL libraries',
     license="MIT",