Merge pull request #502 from int-brain-lab/taskQC_GCaug2022

task QC

Author: juhuntenburg

ibllib/qc/camera.py

@@ -594,7 +594,7 @@ def check_wheel_alignment(self, tolerance=(1, 2), display=False):
 
         # Determine the outcome.  If there are two values for the tolerance, one is taken to be
         # a warning threshold, the other a failure threshold.
-        out_map = {0: 'FAIL', 1: 'WARNING', 2: 'PASS'}
+        out_map = {0: 'WARNING', 1: 'WARNING', 2: 'PASS'}  # 0: FAIL -> WARNING Aug 2022
         passed = np.abs(offset) <= np.sort(np.array(tolerance))
         return out_map[sum(passed)], int(offset)
 

ibllib/qc/dlc.py

@@ -46,10 +46,11 @@ class DlcQC(base.QC):
         'body': ['_ibl_bodyCamera.dlc.*', '_ibl_bodyCamera.times.*'],
     }
 
-    def __init__(self, session_path_or_eid, side, **kwargs):
+    def __init__(self, session_path_or_eid, side, ignore_checks=['check_pupil_diameter_snr'], **kwargs):
         """
         :param session_path_or_eid: A session eid or path
         :param side: The camera to run QC on
+        :param ignore_checks: Checks that won't count towards aggregate QC, but will be run and added to extended QC
         :param log: A logging.Logger instance, if None the 'ibllib' logger is used
         :param one: An ONE instance for fetching and setting the QC on Alyx
         """
@@ -61,6 +62,8 @@ def __init__(self, session_path_or_eid, side, **kwargs):
         super().__init__(session_path_or_eid, **kwargs)
         self.data = Bunch()
 
+        # checks to be added to extended QC but not taken into account for aggregate QC
+        self.ignore_checks = ignore_checks
         # QC outcomes map
         self.metrics = None
 
@@ -139,9 +142,9 @@ def is_metric(x):
         checks = getmembers(DlcQC, is_metric)
         self.metrics = {f'_{namespace}_' + k[6:]: fn(self) for k, fn in checks}
 
-        values = [x if isinstance(x, str) else x[0] for x in self.metrics.values()]
-        code = max(base.CRITERIA[x] for x in values)
-        outcome = next(k for k, v in base.CRITERIA.items() if v == code)
+        ignore_metrics = [f'_{namespace}_' + i[6:] for i in self.ignore_checks]
+        metrics_to_aggregate = {k: v for k, v in self.metrics.items() if k not in ignore_metrics}
+        outcome = self.overall_outcome(metrics_to_aggregate.values())
 
         if update:
             extended = {

ibllib/qc/task_metrics.py

@@ -61,12 +61,23 @@
 
 class TaskQC(base.QC):
     """A class for computing task QC metrics"""
-    criteria = {"PASS": 0.99, "WARNING": 0.95, "FAIL": 0}
-    fcns_value2status = {'default': lambda x: TaskQC._thresholding(x),
-                         '_task_stimFreeze_delays': lambda x: - 1,
-                         '_task_response_stimFreeze_delays': lambda x: -1,
-                         '_task_passed_trial_checks': lambda x: -1,
-                         '_task_iti_delays': lambda x: -1}
+
+    criteria = dict()
+    criteria['default'] = {"PASS": 0.99, "WARNING": 0.90, "FAIL": 0}  # Note: WARNING was 0.95 prior to Aug 2022
+    criteria['_task_stimOff_itiIn_delays'] = {"PASS": 0.99, "WARNING": 0}
+    criteria['_task_positive_feedback_stimOff_delays'] = {"PASS": 0.99, "WARNING": 0}
+    criteria['_task_negative_feedback_stimOff_delays'] = {"PASS": 0.99, "WARNING": 0}
+    criteria['_task_wheel_move_during_closed_loop'] = {"PASS": 0.99, "WARNING": 0}
+    criteria['_task_response_stimFreeze_delays'] = {"PASS": 0.99, "WARNING": 0}
+    criteria['_task_detected_wheel_moves'] = {"PASS": 0.99, "WARNING": 0}
+    criteria['_task_trial_length'] = {"PASS": 0.99, "WARNING": 0}
+    criteria['_task_goCue_delays'] = {"PASS": 0.99, "WARNING": 0}
+    criteria['_task_errorCue_delays'] = {"PASS": 0.99, "WARNING": 0}
+    criteria['_task_stimOn_delays'] = {"PASS": 0.99, "WARNING": 0}
+    criteria['_task_stimOff_delays'] = {"PASS": 0.99, "WARNING": 0}
+    criteria['_task_stimFreeze_delays'] = {"PASS": 0.99, "WARNING": 0}
+    criteria['_task_iti_delays'] = {"NOT_SET": 0}
+    criteria['_task_passed_trial_checks'] = {"NOT_SET": 0}
 
     @staticmethod
     def _thresholding(qc_value, thresholds=None):
@@ -75,18 +86,25 @@ def _thresholding(qc_value, thresholds=None):
         :param qc_value: proportion of passing qcs, between 0 and 1
         :param thresholds: dictionary with keys 'PASS', 'WARNING', 'FAIL'
          (cf. TaskQC.criteria attribute)
-        :return: int where -1: NOT_SET, 0: FAIL, 1: WARNING, 2: PASS
+        :return: int where -1: NOT_SET, 0: PASS, 1: WARNING, 2: FAIL
         """
         MAX_BOUND, MIN_BOUND = (1, 0)
         if not thresholds:
-            thresholds = TaskQC.criteria.copy()
+            thresholds = TaskQC.criteria['default'].copy()
         if qc_value is None or np.isnan(qc_value):
             return int(-1)
         elif (qc_value > MAX_BOUND) or (qc_value < MIN_BOUND):
             raise ValueError("Values out of bound")
-        else:
-            passed = qc_value >= np.fromiter(thresholds.values(), dtype=float)
-            return int(np.argmax(passed))
+        if 'PASS' in thresholds.keys() and qc_value >= thresholds['PASS']:
+            return 0
+        if 'WARNING' in thresholds.keys() and qc_value >= thresholds['WARNING']:
+            return 1
+        if 'FAIL' in thresholds and qc_value >= thresholds['FAIL']:
+            return 2
+        if 'NOT_SET' in thresholds and qc_value >= thresholds['NOT_SET']:
+            return -1
+        # if None of this applies, return 'NOT_SET'
+        return -1
 
     def __init__(self, session_path_or_eid, **kwargs):
         """
@@ -163,16 +181,15 @@ def compute_session_status_from_dict(results):
         :return: Overall session QC outcome as a string
         :return: A dict of QC tests and their outcomes
         """
-        v2status_fcns = TaskQC.fcns_value2status  # the need to have this as a parameter may arise
         indices = np.zeros(len(results), dtype=int)
         for i, k in enumerate(results):
-            if k in v2status_fcns:
-                indices[i] = v2status_fcns[k](results[k])
+            if k in TaskQC.criteria.keys():
+                indices[i] = TaskQC._thresholding(results[k], thresholds=TaskQC.criteria[k])
             else:
-                indices[i] = v2status_fcns['default'](results[k])
+                indices[i] = TaskQC._thresholding(results[k], thresholds=TaskQC.criteria['default'])
 
         def key_map(x):
-            return 'NOT_SET' if x < 0 else list(TaskQC.criteria.keys())[x]
+            return 'NOT_SET' if x < 0 else list(TaskQC.criteria['default'].keys())[x]
         # Criteria map is in order of severity so the max index is our overall QC outcome
         session_outcome = key_map(max(indices))
         outcomes = dict(zip(results.keys(), map(key_map, indices)))

ibllib/tests/qc/test_task_metrics.py

@@ -13,13 +13,38 @@
 
 
 class TestAggregateOutcome(unittest.TestCase):
-    def test_outcome_from_dict(self):
+    def test_outcome_from_dict_default(self):
+        # For a task that has no costume thresholds, default is 0.99 PASS and 0.9 WARNING and 0 FAIL,
+        # np.nan and None return not set
+        qc_dict = {'gnap': .99, 'gnop': np.nan, 'gnip': None, 'gnep': 0.9, 'gnup': 0.89}
+        expect = {'gnap': 'PASS', 'gnop': 'NOT_SET', 'gnip': 'NOT_SET', 'gnep': 'WARNING', 'gnup': 'FAIL'}
+        outcome, outcome_dict = qcmetrics.TaskQC.compute_session_status_from_dict(qc_dict)
+        self.assertEqual(outcome, 'FAIL')
+        self.assertEqual(expect, outcome_dict)
+
+    def test_outcome_from_dict_stimFreeze_delays(self):
+        # For '_task_stimFreeze_delays' the threshold are 0.99 PASS and 0 WARNING
         qc_dict = {'gnap': .99, 'gnop': np.nan, '_task_stimFreeze_delays': .1}
-        expect = {'gnap': 'PASS', 'gnop': 'NOT_SET', '_task_stimFreeze_delays': 'NOT_SET'}
+        expect = {'gnap': 'PASS', 'gnop': 'NOT_SET', '_task_stimFreeze_delays': 'WARNING'}
+        outcome, outcome_dict = qcmetrics.TaskQC.compute_session_status_from_dict(qc_dict)
+        self.assertEqual(outcome, 'WARNING')
+        self.assertEqual(expect, outcome_dict)
+
+    def test_outcome_from_dict_iti_delays(self):
+        # For '_task_iti_delays' the threshold is 0 NOT_SET
+        qc_dict = {'gnap': .99, 'gnop': np.nan, '_task_iti_delays': .1}
+        expect = {'gnap': 'PASS', 'gnop': 'NOT_SET', '_task_iti_delays': 'NOT_SET'}
         outcome, outcome_dict = qcmetrics.TaskQC.compute_session_status_from_dict(qc_dict)
         self.assertEqual(outcome, 'PASS')
         self.assertEqual(expect, outcome_dict)
 
+    def test_out_of_bounds(self):
+        # When qc values are below 0 or above 1, give error
+        qc_dict = {'gnap': 1.01, 'gnop': 0, 'gnip': 0.99}
+        with self.assertRaises(ValueError) as e:
+            outcome, outcome_dict = qcmetrics.TaskQC.compute_session_status_from_dict(qc_dict)
+        self.assertTrue(e.exception.args[0] == 'Values out of bound')
+
 
 class TestTaskMetrics(unittest.TestCase):
     def setUp(self):