Merge pull request #704 from int-brain-lab/task_qc_viewer

Task qc viewer

Author: k1o0

brainbox/behavior/dlc.py

@@ -1,6 +1,4 @@
-"""
-Set of functions to deal with dlc data
-"""
+"""Set of functions to deal with dlc data."""
 import logging
 import pandas as pd
 import warnings
@@ -48,21 +46,23 @@ def insert_idx(array, values):
 
 def likelihood_threshold(dlc, threshold=0.9):
     """
-    Set dlc points with likelihood less than threshold to nan
+    Set dlc points with likelihood less than threshold to nan.
+
+    FIXME Add unit test.
     :param dlc: dlc pqt object
     :param threshold: likelihood threshold
     :return:
     """
     features = np.unique(['_'.join(x.split('_')[:-1]) for x in dlc.keys()])
     for feat in features:
         nan_fill = dlc[f'{feat}_likelihood'] < threshold
-        dlc.loc[nan_fill, f'{feat}_x'] = np.nan
-        dlc.loc[nan_fill, f'{feat}_y'] = np.nan
+        dlc.loc[nan_fill, (f'{feat}_x', f'{feat}_y')] = np.nan
     return dlc
 
 
 def get_speed(dlc, dlc_t, camera, feature='paw_r'):
     """
+    FIXME Document and add unit test!
 
     :param dlc: dlc pqt table
     :param dlc_t: dlc time points

ibllib/io/raw_data_loaders.py

@@ -1,7 +1,3 @@
-#!/usr/bin/env python
-# -*- coding:utf-8 -*-
-# @Author: Niccolò Bonacchi, Miles Wells
-# @Date: Monday, July 16th 2018, 1:28:46 pm
 """
 Raw Data Loader functions for PyBpod rig.
 

ibllib/misc/qt.py

@@ -0,0 +1,45 @@
+"""PyQt5 helper functions."""
+import logging
+import sys
+from functools import wraps
+
+from PyQt5 import QtWidgets
+
+_logger = logging.getLogger(__name__)
+
+
+def get_main_window():
+    """Get the Main window of a QT application."""
+    app = QtWidgets.QApplication.instance()
+    return [w for w in app.topLevelWidgets() if isinstance(w, QtWidgets.QMainWindow)][0]
+
+
+def create_app():
+    """Create a Qt application."""
+    global QT_APP
+    QT_APP = QtWidgets.QApplication.instance()
+    if QT_APP is None:  # pragma: no cover
+        QT_APP = QtWidgets.QApplication(sys.argv)
+    return QT_APP
+
+
+def require_qt(func):
+    """Function decorator to specify that a function requires a Qt application.
+
+    Use this decorator to specify that a function needs a running Qt application before it can run.
+    An error is raised if that is not the case.
+    """
+    @wraps(func)
+    def wrapped(*args, **kwargs):
+        if not QtWidgets.QApplication.instance():
+            _logger.warning('Creating a Qt application.')
+            create_app()
+        return func(*args, **kwargs)
+    return wrapped
+
+
+@require_qt
+def run_app():  # pragma: no cover
+    """Run the Qt application."""
+    global QT_APP
+    return QT_APP.exit(QT_APP.exec_())

ibllib/pipes/base_tasks.py

@@ -75,6 +75,9 @@ def read_params_file(self):
 
 class BehaviourTask(DynamicTask):
 
+    extractor = None
+    """ibllib.io.extractors.base.BaseBpodExtractor: A trials extractor object."""
+
     def __init__(self, session_path, **kwargs):
         super().__init__(session_path, **kwargs)
 
@@ -207,6 +210,70 @@ def _spacer_support(settings):
         ver = v(settings.get('IBLRIG_VERSION') or '100.0.0')
         return ver not in (v('100.0.0'), v('8.0.0')) and ver >= v('7.1.0')
 
+    def extract_behaviour(self, save=True):
+        """Extract trials data.
+
+        This is an abstract method called by `_run` and `run_qc` methods.  Subclasses should return
+        the extracted trials data and a list of output files. This method should also save the
+        trials extractor object to the :prop:`extractor` property for use by `run_qc`.
+
+        Parameters
+        ----------
+        save : bool
+            Whether to save the extracted data as ALF datasets.
+
+        Returns
+        -------
+        dict
+            A dictionary of trials data.
+        list of pathlib.Path
+            A list of output file paths if save == true.
+        """
+        return None, None
+
+    def run_qc(self, trials_data=None, update=True):
+        """Run task QC.
+
+        Subclass method should return the QC object. This just validates the trials_data is not
+        None.
+
+        Parameters
+        ----------
+        trials_data : dict
+            A dictionary of extracted trials data. The output of :meth:`extract_behaviour`.
+        update : bool
+            If true, update Alyx with the QC outcome.
+
+        Returns
+        -------
+        ibllib.qc.task_metrics.TaskQC
+            A TaskQC object replete with task data and computed metrics.
+        """
+        self._assert_trials_data(trials_data)
+        return None
+
+    def _assert_trials_data(self, trials_data=None):
+        """Check trials data available.
+
+        Called by :meth:`run_qc`, this extracts the trial data if `trials_data` is None, and raises
+        if :meth:`extract_behaviour` returns None.
+
+        Parameters
+        ----------
+        trials_data : dict, None
+            A dictionary of extracted trials data or None.
+
+        Returns
+        -------
+        trials_data : dict
+            A dictionary of extracted trials data. The output of :meth:`extract_behaviour`.
+        """
+        if not self.extractor or trials_data is None:
+            trials_data, _ = self.extract_behaviour(save=False)
+        if not (trials_data and self.extractor):
+            raise ValueError('No trials data and/or extractor found')
+        return trials_data
+
 
 class VideoTask(DynamicTask):
 

ibllib/pipes/behavior_tasks.py

@@ -75,27 +75,24 @@ def _run(self, update=True, save=True):
         """
         Extracts an iblrig training session
         """
-        trials, output_files = self._extract_behaviour(save=save)
+        trials, output_files = self.extract_behaviour(save=save)
 
         if trials is None:
             return None
         if self.one is None or self.one.offline:
             return output_files
 
         # Run the task QC
-        self._run_qc(trials, update=update)
+        self.run_qc(trials, update=update)
         return output_files
 
-    def _extract_behaviour(self, **kwargs):
+    def extract_behaviour(self, **kwargs):
         self.extractor = get_bpod_extractor(self.session_path, task_collection=self.collection)
         self.extractor.default_path = self.output_collection
         return self.extractor.extract(task_collection=self.collection, **kwargs)
 
-    def _run_qc(self, trials_data=None, update=True):
-        if not self.extractor or trials_data is None:
-            trials_data, _ = self._extract_behaviour(save=False)
-        if not trials_data:
-            raise ValueError('No trials data found')
+    def run_qc(self, trials_data=None, update=True):
+        trials_data = self._assert_trials_data(trials_data)  # validate trials data
 
         # Compile task data for QC
         qc = HabituationQC(self.session_path, one=self.one)
@@ -130,10 +127,10 @@ def signature(self):
             ('*.meta', self.sync_collection, True)]
         return signature
 
-    def _extract_behaviour(self, save=True, **kwargs):
+    def extract_behaviour(self, save=True, **kwargs):
         """Extract the habituationChoiceWorld trial data using NI DAQ clock."""
         # Extract Bpod trials
-        bpod_trials, _ = super()._extract_behaviour(save=False, **kwargs)
+        bpod_trials, _ = super().extract_behaviour(save=False, **kwargs)
 
         # Sync Bpod trials to FPGA
         sync, chmap = get_sync_and_chn_map(self.session_path, self.sync_collection)
@@ -146,13 +143,13 @@ def _extract_behaviour(self, save=True, **kwargs):
             task_collection=self.collection, protocol_number=self.protocol_number, **kwargs)
         return outputs, files
 
-    def _run_qc(self, trials_data=None, update=True, **_):
+    def run_qc(self, trials_data=None, update=True, **_):
         """Run and update QC.
 
         This adds the bpod TTLs to the QC object *after* the QC is run in the super call method.
         The raw Bpod TTLs are not used by the QC however they are used in the iblapps QC plot.
         """
-        qc = super()._run_qc(trials_data=trials_data, update=update)
+        qc = super().run_qc(trials_data=trials_data, update=update)
         qc.extractor.bpod_ttls = self.extractor.bpod
         return qc
 
@@ -300,26 +297,24 @@ def signature(self):
         return signature
 
     def _run(self, update=True, save=True):
-        """
-        Extracts an iblrig training session
-        """
-        trials, output_files = self._extract_behaviour(save=save)
+        """Extracts an iblrig training session."""
+        trials, output_files = self.extract_behaviour(save=save)
         if trials is None:
             return None
         if self.one is None or self.one.offline:
             return output_files
 
         # Run the task QC
-        self._run_qc(trials)
+        self.run_qc(trials)
 
         return output_files
 
-    def _extract_behaviour(self, **kwargs):
+    def extract_behaviour(self, **kwargs):
         self.extractor = get_bpod_extractor(self.session_path, task_collection=self.collection)
         self.extractor.default_path = self.output_collection
         return self.extractor.extract(task_collection=self.collection, **kwargs)
 
-    def _run_qc(self, trials_data=None, update=True, QC=None):
+    def run_qc(self, trials_data=None, update=True, QC=None):
         """
         Run the task QC.
 
@@ -337,10 +332,7 @@ def _run_qc(self, trials_data=None, update=True, QC=None):
         ibllib.qc.task_metrics.TaskQC
             The task QC object.
         """
-        if not self.extractor or trials_data is None:
-            trials_data, _ = self._extract_behaviour(save=False)
-        if not trials_data:
-            raise ValueError('No trials data found')
+        trials_data = self._assert_trials_data(trials_data)  # validate trials data
 
         # Compile task data for QC
         qc_extractor = TaskQCExtractor(self.session_path, lazy=True, sync_collection=self.sync_collection, one=self.one,
@@ -419,9 +411,9 @@ def _behaviour_criterion(self, update=True, truncate_to_pass=True):
                 "sessions", eid, "extended_qc", {"behavior": int(good_enough)}
             )
 
-    def _extract_behaviour(self, save=True, **kwargs):
+    def extract_behaviour(self, save=True, **kwargs):
         # Extract Bpod trials
-        bpod_trials, _ = super()._extract_behaviour(save=False, **kwargs)
+        bpod_trials, _ = super().extract_behaviour(save=False, **kwargs)
 
         # Sync Bpod trials to FPGA
         sync, chmap = get_sync_and_chn_map(self.session_path, self.sync_collection)
@@ -431,11 +423,8 @@ def _extract_behaviour(self, save=True, **kwargs):
             task_collection=self.collection, protocol_number=self.protocol_number, **kwargs)
         return outputs, files
 
-    def _run_qc(self, trials_data=None, update=False, plot_qc=False, QC=None):
-        if not self.extractor or trials_data is None:
-            trials_data, _ = self._extract_behaviour(save=False)
-        if not trials_data:
-            raise ValueError('No trials data found')
+    def run_qc(self, trials_data=None, update=False, plot_qc=False, QC=None):
+        trials_data = self._assert_trials_data(trials_data)  # validate trials data
 
         # Compile task data for QC
         qc_extractor = TaskQCExtractor(self.session_path, lazy=True, sync_collection=self.sync_collection, one=self.one,
@@ -477,13 +466,13 @@ def _run_qc(self, trials_data=None, update=False, plot_qc=False, QC=None):
         return qc
 
     def _run(self, update=True, plot_qc=True, save=True):
-        dsets, out_files = self._extract_behaviour(save=save)
+        dsets, out_files = self.extract_behaviour(save=save)
 
         if not self.one or self.one.offline:
             return out_files
 
         self._behaviour_criterion(update=update)
-        self._run_qc(dsets, update=update, plot_qc=plot_qc)
+        self.run_qc(dsets, update=update, plot_qc=plot_qc)
         return out_files
 
 
@@ -508,10 +497,10 @@ def signature(self):
                                              for fn in filter(None, extractor.save_names)]
         return signature
 
-    def _extract_behaviour(self, save=True, **kwargs):
+    def extract_behaviour(self, save=True, **kwargs):
         """Extract the Bpod trials data and Timeline acquired signals."""
         # First determine the extractor from the task protocol
-        bpod_trials, _ = ChoiceWorldTrialsBpod._extract_behaviour(self, save=False, **kwargs)
+        bpod_trials, _ = ChoiceWorldTrialsBpod.extract_behaviour(self, save=False, **kwargs)
 
         # Sync Bpod trials to DAQ
         self.extractor = TimelineTrials(self.session_path, bpod_trials=bpod_trials, bpod_extractor=self.extractor)
@@ -544,11 +533,12 @@ def signature(self):
 
     def _run(self, upload=True):
         """
-        Extracts training status for subject
+        Extracts training status for subject.
         """
 
         lab = get_lab(self.session_path, self.one.alyx)
-        if lab == 'cortexlab':
+        if lab == 'cortexlab' and 'cortexlab' in self.one.alyx.base_url:
+            _logger.info('Switching from cortexlab Alyx to IBL Alyx for training status queries.')
             one = ONE(base_url='https://alyx.internationalbrainlab.org')
         else:
             one = self.one