Merge pull request #627 from int-brain-lab/photometry_witten

Photometry witten

Author: mayofaulkner

ibllib/io/extractors/fibrephotometry.py

@@ -46,15 +46,22 @@
 NEUROPHOTOMETRICS_LED_STATES = {
     'Condition': {
         0: 'No additional signal',
-        1: 'Output 0 signal HIGH + Stimulation',
-        2: 'Output 0 signal HIGH + Input 0 signal HIGH',
-        3: 'Input 0 signal HIGH + Stimulation',
-        4: 'Output 0 HIGH + Input 0 HIGH + Stimulation'
+        1: 'Output 1 signal HIGH',
+        2: 'Output 0 signal HIGH',
+        3: 'Stimulation ON',
+        4: 'GPIO Line 2 HIGH',
+        5: 'GPIO Line 3 HIGH',
+        6: 'Input 1 HIGH',
+        7: 'Input 0 HIGH',
+        8: 'Output 0 signal HIGH + Stimulation',
+        9: 'Output 0 signal HIGH + Input 0 signal HIGH',
+        10: 'Input 0 signal HIGH + Stimulation',
+        11: 'Output 0 HIGH + Input 0 HIGH + Stimulation',
     },
-    'No LED ON': {0: 0, 1: 48, 2: 528, 3: 544, 4: 560},
-    'L415': {0: 1, 1: 49, 2: 529, 3: 545, 4: 561},
-    'L470': {0: 2, 1: 50, 2: 530, 3: 546, 4: 562},
-    'L560': {0: 4, 1: 52, 2: 532, 3: 548, 4: 564}
+    'No LED ON': {0: 0, 1: 8, 2: 16, 3: 32, 4: 64, 5: 128, 6: 256, 7: 512, 8: 48, 9: 528, 10: 544, 11: 560},
+    'L415': {0: 1, 1: 9, 2: 17, 3: 33, 4: 65, 5: 129, 6: 257, 7: 513, 8: 49, 9: 529, 10: 545, 11: 561},
+    'L470': {0: 2, 1: 10, 2: 18, 3: 34, 4: 66, 5: 130, 6: 258, 7: 514, 8: 50, 9: 530, 10: 546, 11: 562},
+    'L560': {0: 4, 1: 12, 2: 20, 3: 36, 4: 68, 5: 132, 6: 260, 7: 516, 8: 52, 9: 532, 10: 548, 11: 564}
 }
 
 

ibllib/pipes/behavior_tasks.py

@@ -5,7 +5,9 @@
 from pkg_resources import parse_version
 import one.alf.io as alfio
 from one.alf.files import session_path_parts
+from one.api import ONE
 
+from ibllib.oneibl.registration import get_lab
 from ibllib.pipes import base_tasks
 from ibllib.io.raw_data_loaders import load_settings
 from ibllib.qc.task_extractors import TaskQCExtractor
@@ -455,7 +457,14 @@ def _run(self, upload=True):
         """
         Extracts training status for subject
         """
-        df = training_status.get_latest_training_information(self.session_path, self.one)
+
+        lab = get_lab(self.session_path, self.one.alyx)
+        if lab == 'cortexlab':
+            one = ONE(base_url='https://alyx.internationalbrainlab.org')
+        else:
+            one = self.one
+
+        df = training_status.get_latest_training_information(self.session_path, one)
         if df is not None:
             training_status.make_plots(self.session_path, self.one, df=df, save=True, upload=upload)
             # Update status map in JSON field of subjects endpoint

ibllib/pipes/dynamic_pipeline.py

@@ -19,7 +19,8 @@
 import ibllib.pipes.video_tasks as vtasks
 import ibllib.pipes.ephys_tasks as etasks
 import ibllib.pipes.audio_tasks as atasks
-from ibllib.pipes.photometry_tasks import TaskFibrePhotometryPreprocess, TaskFibrePhotometryRegisterRaw
+import ibllib.pipes.photometry_tasks as ptasks
+# from ibllib.pipes.photometry_tasks import FibrePhotometryPreprocess, FibrePhotometryRegisterRaw
 
 _logger = logging.getLogger(__name__)
 
@@ -307,9 +308,9 @@ def make_pipeline(session_path, **pkwargs):
                 tasks[tn] = type((tn := f'VideoSyncQC_{sync}'), (vtasks.VideoSyncQcNidq,), {})(
                     **kwargs, **video_kwargs, **sync_kwargs, parents=[tasks['VideoCompress']] + sync_tasks)
 
-        if len(video_kwargs['cameras']) == 3:
-            tasks[tn] = type((tn := 'DLC'), (epp.EphysDLC,), {})(
-                **kwargs, parents=[dlc_parent_task])
+        if sync_kwargs['sync'] != 'bpod':
+            tasks[tn] = type((tn := 'DLC'), (vtasks.DLC,), {})(
+                **kwargs, **video_kwargs, parents=[dlc_parent_task])
             tasks['PostDLC'] = type('PostDLC', (epp.EphysPostDLC,), {})(
                 **kwargs, parents=[tasks['DLC'], tasks[f'VideoSyncQC_{sync}']])
 
@@ -357,11 +358,11 @@ def make_pipeline(session_path, **pkwargs):
     if 'photometry' in devices:
         # {'collection': 'raw_photometry_data', 'sync_label': 'frame_trigger', 'regions': ['Region1G', 'Region3G']}
         photometry_kwargs = devices['photometry']
-        tasks['TaskFibrePhotometryRegisterRaw'] = type('TaskFibrePhotometryRegisterRaw', (
-            TaskFibrePhotometryRegisterRaw,), {})(**kwargs, **photometry_kwargs)
-        tasks['TaskFibrePhotometryPreprocess'] = type('TaskFibrePhotometryPreprocess', (
-            TaskFibrePhotometryPreprocess,), {})(**kwargs, **photometry_kwargs, **sync_kwargs,
-                                                 parents=[tasks['TaskFibrePhotometryRegisterRaw']] + sync_tasks)
+        tasks['FibrePhotometryRegisterRaw'] = type('FibrePhotometryRegisterRaw', (
+            ptasks.FibrePhotometryRegisterRaw,), {})(**kwargs, **photometry_kwargs)
+        tasks['FibrePhotometryPreprocess'] = type('FibrePhotometryPreprocess', (
+            ptasks.FibrePhotometryPreprocess,), {})(**kwargs, **photometry_kwargs, **sync_kwargs,
+                                                    parents=[tasks['FibrePhotometryRegisterRaw']] + sync_tasks)
 
     p = mtasks.Pipeline(session_path=session_path, **pkwargs)
     p.tasks = tasks

ibllib/pipes/ephys_preprocessing.py

@@ -21,8 +21,7 @@
 from ibllib.io.video import label_from_path, assert_valid_label
 from ibllib.io.extractors import ephys_fpga, ephys_passive, camera
 from ibllib.pipes import tasks, base_tasks
-from ibllib.pipes.training_preprocessing import TrainingRegisterRaw as EphysRegisterRaw
-from ibllib.pipes.training_preprocessing import TrainingStatus as EphysTrainingStatus
+import ibllib.pipes.training_preprocessing as tpp
 from ibllib.pipes.misc import create_alyx_probe_insertions
 from ibllib.qc.alignment_qc import get_aligned_channels
 from ibllib.qc.task_extractors import TaskQCExtractor
@@ -1324,7 +1323,7 @@ def __init__(self, session_path=None, **kwargs):
         self.session_path = session_path
         # level 0
         tasks['ExperimentDescriptionRegisterRaw'] = base_tasks.ExperimentDescriptionRegisterRaw(self.session_path)
-        tasks["EphysRegisterRaw"] = EphysRegisterRaw(self.session_path)
+        tasks["EphysRegisterRaw"] = tpp.TrainingRegisterRaw(self.session_path)
         tasks["EphysPulses"] = EphysPulses(self.session_path)
         tasks["EphysRawQC"] = RawEphysQC(self.session_path)
         tasks["EphysAudio"] = EphysAudio(self.session_path)
@@ -1341,7 +1340,7 @@ def __init__(self, session_path=None, **kwargs):
             self.session_path, parents=[tasks["EphysVideoCompress"], tasks["EphysPulses"], tasks["EphysTrials"]])
         tasks["EphysCellsQc"] = EphysCellsQc(self.session_path, parents=[tasks["SpikeSorting"]])
         tasks["EphysDLC"] = EphysDLC(self.session_path, parents=[tasks["EphysVideoCompress"]])
-        tasks['EphysTrainingStatus'] = EphysTrainingStatus(self.session_path, parents=[tasks["EphysTrials"]])
+        tasks['EphysTrainingStatus'] = tpp.TrainingStatus(self.session_path, parents=[tasks["EphysTrials"]])
         # level 3
         tasks["EphysPostDLC"] = EphysPostDLC(self.session_path, parents=[tasks["EphysDLC"], tasks["EphysTrials"],
                                                                          tasks["EphysVideoSyncQc"]])

ibllib/pipes/photometry_tasks.py

@@ -4,49 +4,55 @@
 from collections import OrderedDict
 
 from ibllib.pipes import tasks, base_tasks
-from ibllib.pipes.training_preprocessing import (
-    TrainingRegisterRaw, TrainingAudio, TrainingTrials, TrainingDLC, TrainingStatus, TrainingVideoCompress)
+import ibllib.pipes.training_preprocessing as tpp
 from ibllib.io.extractors.fibrephotometry import FibrePhotometry
 
 _logger = logging.getLogger('ibllib')
 
 
-class TaskFibrePhotometryRegisterRaw(base_tasks.RegisterRawDataTask):
+class FibrePhotometryRegisterRaw(base_tasks.RegisterRawDataTask):
 
     priority = 100
     job_size = 'small'
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.collection = self.get_task_collection(kwargs.get('collection', None))
+        self.device_collection = self.get_device_collection('photometry', device_collection='raw_photometry_data')
 
     @property
     def signature(self):
         signature = {
             'input_files': [],
-            'output_files': [('_mcc_DAQdata.raw.tdms', self.collection, True),
-                             ('_neurophotometrics_fpData.raw.pqt', self.collection, True)]
+            'output_files': [('_mcc_DAQdata.raw.tdms', self.device_collection, True),
+                             ('_neurophotometrics_fpData.raw.pqt', self.device_collection, True)]
         }
         return signature
 
 
-class TaskFibrePhotometryPreprocess(base_tasks.DynamicTask):
-    signature = {
-        'input_files': [('*fpData.raw*', 'raw_photometry_data', True), ],
-        'output_files': [('photometry.signal.pqt', 'alf', True), ]
-    }
+class FibrePhotometryPreprocess(base_tasks.DynamicTask):
+    @property
+    def signature(self):
+        signature = {
+            'input_files': [('_mcc_DAQdata.raw.tdms', self.device_collection, True),
+                            ('_neurophotometrics_fpData.raw.pqt', self.device_collection, True)],
+            'output_files': [('photometry.signal.pqt', 'alf/photometry', True)]
+        }
+        return signature
+
     priority = 90
     level = 1
 
     def __init__(self, session_path, regions=None, **kwargs):
         super().__init__(session_path, **kwargs)
         # Task collection (this needs to be specified in the task kwargs)
         self.collection = self.get_task_collection(kwargs.get('collection', None))
+        self.device_collection = self.get_device_collection('photometry', device_collection='raw_photometry_data')
         self.regions = regions
 
     def _run(self, **kwargs):
-        _, out_files = FibrePhotometry(self.session_path, collection=self.collection).extract(
-            regions=self.regions, save=True)
+        _, out_files = FibrePhotometry(self.session_path, collection=self.device_collection).extract(
+            regions=self.regions, path_out=self.session_path.joinpath('alf', 'photometry'), save=True)
         return out_files
 
 
@@ -63,13 +69,13 @@ def __init__(self, session_path=None, **kwargs):
         tasks = OrderedDict()
         self.session_path = session_path
         # level 0
-        tasks['TrainingRegisterRaw'] = TrainingRegisterRaw(self.session_path)
-        tasks['TrainingTrials'] = TrainingTrials(self.session_path)
-        tasks['TrainingVideoCompress'] = TrainingVideoCompress(self.session_path)
-        tasks['TrainingAudio'] = TrainingAudio(self.session_path)
+        tasks['TrainingRegisterRaw'] = tpp.TrainingRegisterRaw(self.session_path)
+        tasks['TrainingTrials'] = tpp.TrainingTrials(self.session_path)
+        tasks['TrainingVideoCompress'] = tpp.TrainingVideoCompress(self.session_path)
+        tasks['TrainingAudio'] = tpp.TrainingAudio(self.session_path)
         # level 1
-        tasks['BiasedFibrePhotometry'] = TaskFibrePhotometryPreprocess(self.session_path, parents=[tasks['TrainingTrials']])
-        tasks['TrainingStatus'] = TrainingStatus(self.session_path, parents=[tasks['TrainingTrials']])
-        tasks['TrainingDLC'] = TrainingDLC(
+        tasks['BiasedFibrePhotometry'] = FibrePhotometryPreprocess(self.session_path, parents=[tasks['TrainingTrials']])
+        tasks['TrainingStatus'] = tpp.TrainingStatus(self.session_path, parents=[tasks['TrainingTrials']])
+        tasks['TrainingDLC'] = tpp.TrainingDLC(
             self.session_path, parents=[tasks['TrainingVideoCompress']])
         self.tasks = tasks