Merge tag '2.35.2' into develop

Author: oliche

brainbox/metrics/single_units.py

@@ -980,7 +980,7 @@ def quick_unit_metrics(spike_clusters, spike_times, spike_amps, spike_depths,
     r.amp_median[ir] = np.array(10 ** (camp['log_amps'].median() / 20))
     r.amp_std_dB[ir] = np.array(camp['log_amps'].std())
     srp = metrics.slidingRP_all(spikeTimes=spike_times, spikeClusters=spike_clusters,
-                                **{'sampleRate': 30000, 'binSizeCorr': 1 / 30000})
+                                sampleRate=30000, binSizeCorr=1 / 30000)
     r.slidingRP_viol[ir] = srp['value']
 
     # loop over each cluster to compute the rest of the metrics

brainbox/plot_base.py

@@ -640,7 +640,7 @@ def scatter_xyc_plot(x, y, c, cmap=None, clim=None, rgb=False):
     data.set_clim(clim=clim)
     if rgb:
         norm = matplotlib.colors.Normalize(vmin=data.clim[0], vmax=data.clim[1], clip=True)
-        mapper = cm.ScalarMappable(norm=norm, cmap=cm.get_cmap(cmap))
+        mapper = cm.ScalarMappable(norm=norm, cmap=plt.get_cmap(cmap))
         cluster_color = np.array([mapper.to_rgba(col) for col in c])
         data.set_color(color=cluster_color)
 

ibllib/__init__.py

@@ -2,7 +2,7 @@
 import logging
 import warnings
 
-__version__ = '2.35.1'
+__version__ = '2.35.2'
 warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib')
 
 # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev

ibllib/io/__init__.py

@@ -0,0 +1 @@
+"""Loaders for unprocessed IBL video and task data, and parameter files."""

ibllib/io/extractors/__init__.py

@@ -0,0 +1,4 @@
+"""IBL rig data pre-processing functions.
+
+Extractor classes for loading raw rig data and returning ALF compliant pre-processed data.
+"""

ibllib/io/extractors/training_trials.py

@@ -1,5 +1,6 @@
 import logging
 import numpy as np
+from itertools import accumulate
 from packaging import version
 from one.alf.io import AlfBunch
 
@@ -123,19 +124,16 @@ def _extract(self):
         def get_trial_repeat(trial):
             if 'debias_trial' in trial:
                 return trial['debias_trial']
-            else:
+            elif 'contrast' in trial and isinstance(trial['contrast'], dict):
                 return trial['contrast']['type'] == 'RepeatContrast'
+            else:
+                # For advanced choice world before version 8.19.0 there was no 'debias_trial' field
+                # and no debiasing protocol applied, so simply return False
+                assert self.settings['PYBPOD_PROTOCOL'].startswith('_iblrig_tasks_advancedChoiceWorld')
+                return False
 
-        trial_repeated = np.array(list(map(get_trial_repeat, self.bpod_trials))).astype(int)
-        repNum = trial_repeated.copy()
-        c = 0
-        for i in range(len(trial_repeated)):
-            if trial_repeated[i] == 0:
-                c = 0
-                repNum[i] = 0
-                continue
-            c += 1
-            repNum[i] = c
+        trial_repeated = np.fromiter(map(get_trial_repeat, self.bpod_trials), int)
+        repNum = np.fromiter(accumulate(trial_repeated, lambda x, y: x + y if y else 0), int)
         return repNum
 
 
@@ -163,7 +161,7 @@ class FeedbackTimes(BaseBpodTrialsExtractor):
     **Optional:** saves _ibl_trials.feedback_times.npy
 
     Gets reward  and error state init times vectors,
-    checks if theintersection of nans is empty, then
+    checks if the intersection of nans is empty, then
     merges the 2 vectors.
     """
     save_names = '_ibl_trials.feedback_times.npy'

ibllib/io/extractors/training_wheel.py

@@ -1,3 +1,4 @@
+"""Extractors for the wheel position, velocity, and detected movement."""
 import logging
 from collections.abc import Sized
 
@@ -323,16 +324,35 @@ def extract_wheel_moves(re_ts, re_pos, display=False):
 def extract_first_movement_times(wheel_moves, trials, min_qt=None):
     """
     Extracts the time of the first sufficiently large wheel movement for each trial.
+
     To be counted, the movement must occur between go cue / stim on and before feedback /
     response time.  The movement onset is sometimes just before the cue (occurring in the
     gap between quiescence end and cue start, or during the quiescence period but sub-
-    threshold).  The movement is sufficiently large if it is greater than or equal to THRESH
-    :param wheel_moves: dictionary of detected wheel movement onsets and peak amplitudes for
-    use in extracting each trial's time of first movement.
+    threshold).  The movement is sufficiently large if it is greater than or equal to THRESH.
+
+    :param wheel_moves:
     :param trials: dictionary of trial data
-    :param min_qt: the minimum quiescence period, if None a default is used
-    :return: numpy array of first movement times, bool array indicating whether movement
-    crossed response threshold, and array of indices for wheel_moves arrays
+    :param min_qt:
+    :return: numpy array of
+
+    Parameters
+    ----------
+    wheel_moves : dict
+        Dictionary of detected wheel movement onsets and peak amplitudes for use in extracting each
+        trial's time of first movement.
+    trials : dict
+        Dictionary of trial data.
+    min_qt : float
+        The minimum quiescence period in seconds, if None a default is used.
+
+    Returns
+    -------
+    numpy.array
+        First movement times.
+    numpy.array
+        Bool array indicating whether movement crossed response threshold.
+    numpy.array
+        Indices for wheel_moves arrays.
     """
     THRESH = .1  # peak amp should be at least .1 rad; ~1/3rd of the distance to threshold
     MIN_QT = .2  # default minimum enforced quiescence period
@@ -371,6 +391,8 @@ def extract_first_movement_times(wheel_moves, trials, min_qt=None):
 
 class Wheel(BaseBpodTrialsExtractor):
     """
+    Wheel extractor.
+
     Get wheel data from raw files and converts positions into radians mathematical convention
      (anti-clockwise = +) and timestamps into seconds relative to Bpod clock.
     **Optional:** saves _ibl_wheel.times.npy and _ibl_wheel.position.npy

ibllib/io/extractors/video_motion.py

@@ -42,7 +42,7 @@ def find_nearest(array, value):
 class MotionAlignment:
     roi = {'left': ((800, 1020), (233, 1096)), 'right': ((426, 510), (104, 545)), 'body': ((402, 481), (31, 103))}
 
-    def __init__(self, eid=None, one=None, log=logging.getLogger(__name__), **kwargs):
+    def __init__(self, eid=None, one=None, log=logging.getLogger(__name__), stream=False, **kwargs):
         self.one = one or ONE()
         self.eid = eid
         self.session_path = kwargs.pop('session_path', None) or self.one.eid2path(eid)
@@ -51,7 +51,10 @@ def __init__(self, eid=None, one=None, log=logging.getLogger(__name__), **kwargs
         self.trials = self.wheel = self.camera_times = None
         raw_cam_path = self.session_path.joinpath('raw_video_data')
         camera_path = list(raw_cam_path.glob('_iblrig_*Camera.raw.*'))
-        self.video_paths = {vidio.label_from_path(x): x for x in camera_path}
+        if stream:
+            self.video_paths = vidio.url_from_eid(self.eid)
+        else:
+            self.video_paths = {vidio.label_from_path(x): x for x in camera_path}
         self.data = Bunch()
         self.alignment = Bunch()
 
@@ -107,8 +110,8 @@ def load_data(self, download=False):
         if download:
             self.data.wheel = self.one.load_object(self.eid, 'wheel')
             self.data.trials = self.one.load_object(self.eid, 'trials')
-            cam = self.one.load(self.eid, ['camera.times'], dclass_output=True)
-            self.data.camera_times = {vidio.label_from_path(url): ts for ts, url in zip(cam.data, cam.url)}
+            cam, det = self.one.load_datasets(self.eid, ['*Camera.times*'])
+            self.data.camera_times = {vidio.label_from_path(d['rel_path']): ts for ts, d in zip(cam, det)}
         else:
             alf_path = self.session_path / 'alf'
             wheel_path = next(alf_path.rglob('*wheel.timestamps*')).parent
@@ -329,7 +332,7 @@ def animate(i):
             data['im'].set_data(frame)
 
             mkr = find_nearest(wheel.timestamps[wheel_mask], t_x)
-            data['marker'].set_data(wheel.timestamps[wheel_mask][mkr], wheel.position[wheel_mask][mkr])
+            data['marker'].set_data([wheel.timestamps[wheel_mask][mkr]], [wheel.position[wheel_mask][mkr]])
 
             return data['im'], data['ln'], data['marker']
 

ibllib/oneibl/registration.py

@@ -173,7 +173,7 @@ class IBLRegistrationClient(RegistrationClient):
     Object that keeps the ONE instance and provides method to create sessions and register data.
     """
 
-    def register_session(self, ses_path, file_list=True, projects=None, procedures=None):
+    def register_session(self, ses_path, file_list=True, projects=None, procedures=None, register_reward=True):
         """
         Register an IBL Bpod session in Alyx.
 
@@ -188,11 +188,16 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N
             The project(s) to which the experiment belongs (optional).
         procedures : str, list
             An optional list of procedures, e.g. 'Behavior training/tasks'.
+        register_reward : bool
+            If true, register all water administrations in the settings files, if no admins already
+            present for this session.
 
         Returns
         -------
         dict
             An Alyx session record.
+        list of dict, None
+            Alyx file records (or None if file_list is False).
 
         Notes
         -----
@@ -321,7 +326,7 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N
 
         _logger.info(session['url'] + ' ')
         # create associated water administration if not found
-        if not session['wateradmin_session_related'] and any(task_data):
+        if register_reward and not session['wateradmin_session_related'] and any(task_data):
             for md, d in filter(all, zip(settings, task_data)):
                 _, _end_time = _get_session_times(ses_path, md, d)
                 user = md.get('PYBPOD_CREATOR')

ibllib/tests/extractors/test_extractors.py

@@ -207,21 +207,21 @@ def test_get_choice(self):
             self.assertTrue(all(choice[trial_nogo]) == 0)
 
     def test_get_repNum(self):
-        # TODO: Test its sawtooth
         # TRAINING SESSIONS
         rn = training_trials.RepNum(
             self.training_lt5['path']).extract()[0]
         self.assertTrue(isinstance(rn, np.ndarray))
-        for i in range(3):
-            self.assertTrue(i in rn)
+        expected = [0, 1, 2, 0]
+        np.testing.assert_array_equal(rn, expected)
         # -- version >= 5.0.0
         rn = training_trials.RepNum(
             self.training_ge5['path']).extract()[0]
         self.assertTrue(isinstance(rn, np.ndarray))
-        for i in range(4):
-            self.assertTrue(i in rn)
 
-        # BIASED SESSIONS have no repeted trials
+        expected = [0, 0, 1, 2, 3, 0, 0, 0, 1, 2, 0, 1]
+        np.testing.assert_array_equal(rn, expected)
+
+        # BIASED SESSIONS have no repeated trials
 
     def test_get_rewardVolume(self):
         # TRAINING SESSIONS
@@ -237,14 +237,14 @@ def test_get_rewardVolume(self):
         rv = biased_trials.RewardVolume(
             self.biased_lt5['path']).extract()[0]
         self.assertTrue(isinstance(rv, np.ndarray))
-        # Test if all non zero rewards are of the same value
-        self.assertTrue(all([x == max(rv) for x in rv if x != 0]))
+        # Test if all non-zero rewards are of the same value
+        self.assertTrue(all(x == max(rv) for x in rv if x != 0))
         # -- version >= 5.0.0
         rv = biased_trials.RewardVolume(
             self.biased_ge5['path']).extract()[0]
         self.assertTrue(isinstance(rv, np.ndarray))
-        # Test if all non zero rewards are of the same value
-        self.assertTrue(all([x == max(rv) for x in rv if x != 0]))
+        # Test if all non-zero rewards are of the same value
+        self.assertTrue(all(x == max(rv) for x in rv if x != 0))
 
     def test_get_feedback_times_ge5(self):
         # TRAINING SESSIONS