Merge branch 'trajectory_fix' into protected_view

Author: mayofaulkner

ibllib/__init__.py

@@ -2,7 +2,7 @@
 import logging
 import warnings
 
-__version__ = '2.32.4'
+__version__ = '2.32.5'
 warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib')
 
 # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev

ibllib/io/session_params.py

@@ -169,9 +169,9 @@ def merge_params(a, b, copy=False):
         if k == 'sync':
             assert k not in a or a[k] == b[k], 'multiple sync fields defined'
         if isinstance(b[k], list):
-            prev = a.get(k, [])
+            prev = list(a.get(k, []))
             # For procedures and projects, remove duplicates
-            to_add = b[k] if k == 'tasks' else set(prev) ^ set(b[k])
+            to_add = b[k] if k == 'tasks' else set(b[k]) - set(prev)
             a[k] = prev + list(to_add)
         elif isinstance(b[k], dict):
             a[k] = {**a.get(k, {}), **b[k]}

ibllib/oneibl/patcher.py

@@ -115,7 +115,7 @@ def _patch_dataset(self, path, dset_id=None, revision=None, dry=False, ftp=False
         assert is_uuid_string(dset_id)
         # If the revision is not None then we need to add the revision into the path. Note the moving of the file
         # is handled by one registration client
-        if revision is not None:
+        if revision is not None and f'#{revision}' not in str(path):
             path = path.parent.joinpath(f'#{revision}#', path.name)
         assert path.exists()
         dset = self.one.alyx.rest('datasets', 'read', id=dset_id)

ibllib/oneibl/registration.py

@@ -241,7 +241,7 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N
             # assert len({md['IBLRIG_VERSION_TAG'] for md in settings}) == 1
 
             users = []
-            for user in filter(None, map(lambda x: x.get('PYBPOD_CREATOR'), settings)):
+            for user in filter(lambda x: x and x[1], map(lambda x: x.get('PYBPOD_CREATOR'), settings)):
                 user = self.assert_exists(user[0], 'users')  # user is list of [username, uuid]
                 users.append(user['username'])
 
@@ -266,7 +266,7 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N
             if poo_counts:
                 json_field['POOP_COUNT'] = int(sum(poo_counts))
 
-        if not session:  # Create session and weighings
+        if not len(session):  # Create session and weighings
             ses_ = {'subject': subject['nickname'],
                     'users': users or [subject['responsible_user']],
                     'location': settings[0]['PYBPOD_BOARD'],
@@ -293,9 +293,13 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N
                 self.register_weight(subject['nickname'], md['SUBJECT_WEIGHT'],
                                      date_time=md['SESSION_DATETIME'], user=user)
         else:  # if session exists update a few key fields
-            data = {'procedures': procedures, 'projects': projects}
+            data = {'procedures': procedures, 'projects': projects,
+                    'n_correct_trials': n_correct_trials, 'n_trials': n_trials}
             if task_protocols:
                 data['task_protocol'] = '/'.join(task_protocols)
+            if end_time:
+                data['end_time'] = self.ensure_ISO8601(end_time)
+
             session = self.one.alyx.rest('sessions', 'partial_update', id=session_id[0], data=data)
             if json_field:
                 session['json'] = self.one.alyx.json_field_update('sessions', session['id'], data=json_field)
@@ -421,14 +425,16 @@ def _get_session_times(fn, md, ses_data):
     """
     if isinstance(md, dict):
         start_time = _start_time = isostr2date(md['SESSION_DATETIME'])
+        end_time = isostr2date(md['SESSION_END_TIME']) if md.get('SESSION_END_TIME') else None
     else:
         start_time = isostr2date(md[0]['SESSION_DATETIME'])
         _start_time = isostr2date(md[-1]['SESSION_DATETIME'])
+        end_time = isostr2date(md[-1]['SESSION_END_TIME']) if md[-1].get('SESSION_END_TIME') else None
         assert isinstance(ses_data, (list, tuple)) and len(ses_data) == len(md)
         assert len(md) == 1 or start_time < _start_time
         ses_data = ses_data[-1]
-    if not ses_data:
-        return start_time, None
+    if not ses_data or end_time is not None:
+        return start_time, end_time
     c = ses_duration_secs = 0
     for sd in reversed(ses_data):
         ses_duration_secs = (sd['behavior_data']['Trial end timestamp'] -

ibllib/pipes/histology.py

@@ -396,8 +396,10 @@ def create_trajectory_dict(probe_id, insertion, provenance, endpoint='insertions
              }
     if endpoint == 'chronic-insertions':
         tdict['chronic_insertion'] = probe_id
+        tdict['probe_insertion'] = None
     else:
         tdict['probe_insertion'] = probe_id
+        tdict['chronic_insertion'] = None
 
     return tdict
 

ibllib/qc/alignment_qc.py

@@ -316,7 +316,7 @@ def upload_channels(self, alignment_key, upload_alyx, upload_flatiron):
                 ephys_traj = self.one.alyx.get(f'/trajectories?&probe_insertion={self.eid}'
                                                '&provenance=Ephys aligned histology track',
                                                clobber=True)
-                patch_dict = {'json': self.alignments}
+                patch_dict = {'probe_insertion': self.eid, 'json': self.alignments}
                 self.one.alyx.rest('trajectories', 'partial_update', id=ephys_traj[0]['id'],
                                    data=patch_dict)
 

ibllib/tests/qc/test_alignment_qc.py

@@ -186,6 +186,7 @@ def _02_one_alignment(self):
                       self.alignments['2020-06-26T16:40:14_Karolina_Socha']}
         trajectory = copy.deepcopy(self.trajectory)
         trajectory.update({'json': alignments})
+        trajectory.update({'chronic_insertion': None})
         _ = one.alyx.rest('trajectories', 'create', data=trajectory)
         align_qc = AlignmentQC(self.probe_id, one=one, brain_atlas=brain_atlas, channels=False)
         align_qc.run(update=True, upload_alyx=True, upload_flatiron=False)
@@ -277,6 +278,7 @@ def setUpClass(cls) -> None:
         cls.probe_id = probe_insertion['id']
         cls.trajectory = data['trajectory'].tolist()
         cls.trajectory.update({'probe_insertion': cls.probe_id})
+        cls.trajectory.update({'chronic_insertion': None})
         cls.trajectory.update({'json': cls.alignments})
         cls.traj = one.alyx.rest('trajectories', 'create', data=cls.trajectory)
 
@@ -415,6 +417,7 @@ def setUpClass(cls) -> None:
         cls.probe_name = probe_insertion['name']
         cls.trajectory = data['trajectory'].tolist()
         cls.trajectory.update({'probe_insertion': cls.probe_id})
+        cls.trajectory.update({'chronic_insertion': None})
         cls.trajectory.update({'json': cls.alignments})
         cls.traj = one.alyx.rest('trajectories', 'create', data=cls.trajectory)
 

ibllib/tests/test_io.py

@@ -555,6 +555,23 @@ def test_get_collections_repeat_protocols(self):
         collections = session_params.get_collections(tasks, flat=True)
         self.assertEqual(collections, {'raw_passive_data_bis', 'raw_passive_data', 'raw_behavior_data'})
 
+    def test_merge_params(self):
+        """Test for ibllib.io.session_params.merge_params functions."""
+        a = self.fixture
+        b = {'procedures': ['Imaging', 'Injection'], 'tasks': [{'fooChoiceWorld': {'collection': 'bar'}}]}
+        c = session_params.merge_params(a, b, copy=True)
+        self.assertCountEqual(['Imaging', 'Behavior training/tasks', 'Injection'], c['procedures'])
+        self.assertCountEqual(['passiveChoiceWorld', 'ephysChoiceWorld', 'fooChoiceWorld'], (list(x)[0] for x in c['tasks']))
+        # Ensure a and b not modified
+        self.assertNotEqual(set(c['procedures']), set(a['procedures']))
+        self.assertNotEqual(set(a['procedures']), set(b['procedures']))
+        # Test without copy
+        session_params.merge_params(a, b, copy=False)
+        self.assertCountEqual(['Imaging', 'Behavior training/tasks', 'Injection'], a['procedures'])
+        # Test assertion on duplicate sync
+        b['sync'] = {'foodaq': {'collection': 'raw_sync_data'}}
+        self.assertRaises(AssertionError, session_params.merge_params, a, b)
+
 
 class TestRawDaqLoaders(unittest.TestCase):
     """Tests for raw_daq_loaders module"""

ibllib/tests/test_oneibl.py

@@ -126,7 +126,7 @@ def test_patch_datasets(self):
 
         # Mock the post method of AlyxClient and assert that it was called during registration
         with mock.patch.object(self.one.alyx, 'post') as rest_mock:
-            rest_mock.side_effect = responses
+            rest_mock.side_effect = [[r] for r in responses]
             self.globus_patcher.patch_datasets(file_list)
             self.assertEqual(rest_mock.call_count, 2)
             for call, file in zip(rest_mock.call_args_list, file_list):
@@ -434,16 +434,20 @@ def test_registration_session(self):
                               query_type='remote')[0]
         ses_info = self.one.alyx.rest('sessions', 'read', id=eid)
         self.assertTrue(ses_info['procedures'] == ['Behavior training/tasks'])
-        self.one.alyx.rest('sessions', 'delete', id=eid)
-        # re-register the session as unknown protocol this time
+        # re-register the session as unknown protocol, this time without removing session first
         self.settings['PYBPOD_PROTOCOL'] = 'gnagnagna'
+        # also add an end time
+        start = datetime.datetime.fromisoformat(self.settings['SESSION_DATETIME'])
+        self.settings['SESSION_START_TIME'] = rc.ensure_ISO8601(start)
+        self.settings['SESSION_END_TIME'] = rc.ensure_ISO8601(start + datetime.timedelta(hours=1))
         with open(settings_file, 'w') as fid:
             json.dump(self.settings, fid)
         rc.register_session(self.session_path)
         eid = self.one.search(subject=self.subject, date_range=['2018-04-01', '2018-04-01'],
                               query_type='remote')[0]
         ses_info = self.one.alyx.rest('sessions', 'read', id=eid)
         self.assertTrue(ses_info['procedures'] == [])
+        self.assertEqual(self.settings['SESSION_END_TIME'], ses_info['end_time'])
         self.one.alyx.rest('sessions', 'delete', id=eid)
 
     def test_register_chained_session(self):
@@ -462,12 +466,13 @@ def test_register_chained_session(self):
 
         # Save experiment description
         session_params.write_params(self.session_path, experiment_description)
-
+        self.settings['POOP_COUNT'] = 10
         with open(behaviour_paths[1].joinpath('_iblrig_taskSettings.raw.json'), 'w') as fid:
             json.dump(self.settings, fid)
 
         settings = self.settings.copy()
         settings['PYBPOD_PROTOCOL'] = '_iblrig_tasks_passiveChoiceWorld'
+        settings['POOP_COUNT'] = 53
         start_time = (datetime.datetime.fromisoformat(settings['SESSION_DATETIME']) -
                       datetime.timedelta(hours=1, minutes=2, seconds=12))
         settings['SESSION_DATETIME'] = start_time.isoformat()
@@ -480,7 +485,9 @@ def test_register_chained_session(self):
         ses_info = self.one.alyx.rest('sessions', 'read', id=session['id'])
         self.assertCountEqual(experiment_description['procedures'], ses_info['procedures'])
         self.assertCountEqual(experiment_description['projects'], ses_info['projects'])
-        self.assertCountEqual({'IS_MOCK': False, 'IBLRIG_VERSION': None}, ses_info['json'])
+        # Poo count should be sum of values in both settings files
+        expected = {'IS_MOCK': False, 'IBLRIG_VERSION': '5.4.1', 'POOP_COUNT': 63}
+        self.assertDictEqual(expected, ses_info['json'])
         self.assertEqual('2018-04-01T11:46:14.795526', ses_info['start_time'])
         # Test task protocol
         expected = '_iblrig_tasks_passiveChoiceWorld5.4.1/_iblrig_tasks_ephysChoiceWorld5.4.1'

release_notes.md

@@ -13,6 +13,9 @@
 #### 2.32.4
 - Add support for variations of the biaseCW task in the json task description
 
+#### 2.32.5
+- Minor fixes to IBL registration client, including use of SESSION_END_TIME key
+
 ## Release Notes 2.31
 
 ### features