Merge branch 'develop' into docs

Author: mayofaulkner

brainbox/metrics/single_units.py

@@ -980,7 +980,7 @@ def quick_unit_metrics(spike_clusters, spike_times, spike_amps, spike_depths,
     r.amp_median[ir] = np.array(10 ** (camp['log_amps'].median() / 20))
     r.amp_std_dB[ir] = np.array(camp['log_amps'].std())
     srp = metrics.slidingRP_all(spikeTimes=spike_times, spikeClusters=spike_clusters,
-                                **{'sampleRate': 30000, 'binSizeCorr': 1 / 30000})
+                                sampleRate=30000, binSizeCorr=1 / 30000)
     r.slidingRP_viol[ir] = srp['value']
 
     # loop over each cluster to compute the rest of the metrics

brainbox/plot_base.py

@@ -640,7 +640,7 @@ def scatter_xyc_plot(x, y, c, cmap=None, clim=None, rgb=False):
     data.set_clim(clim=clim)
     if rgb:
         norm = matplotlib.colors.Normalize(vmin=data.clim[0], vmax=data.clim[1], clip=True)
-        mapper = cm.ScalarMappable(norm=norm, cmap=cm.get_cmap(cmap))
+        mapper = cm.ScalarMappable(norm=norm, cmap=plt.get_cmap(cmap))
         cluster_color = np.array([mapper.to_rgba(col) for col in c])
         data.set_color(color=cluster_color)
 

brainbox/population/cca.py

@@ -1,11 +1,14 @@
 import numpy as np
 import matplotlib.pylab as plt
+from scipy.signal.windows import gaussian
+from scipy.signal import convolve
+from sklearn.decomposition import PCA
+
 from iblutil.numerical import bincount2D
 
 
 def _smooth(data, sd):
-    from scipy.signal import gaussian
-    from scipy.signal import convolve
+
     n_bins = data.shape[0]
     w = n_bins - 1 if n_bins % 2 == 0 else n_bins
     window = gaussian(w, std=sd)
@@ -15,7 +18,6 @@ def _smooth(data, sd):
 
 
 def _pca(data, n_pcs):
-    from sklearn.decomposition import PCA
     pca = PCA(n_components=n_pcs)
     pca.fit(data)
     data_pc = pca.transform(data)

brainbox/singlecell.py

@@ -3,7 +3,8 @@
 '''
 
 import numpy as np
-from scipy.signal import convolve, gaussian
+from scipy.signal import convolve
+from scipy.signal.windows import gaussian
 from iblutil.util import Bunch
 from brainbox.population.decode import xcorr
 

ibllib/__init__.py

@@ -2,7 +2,7 @@
 import logging
 import warnings
 
-__version__ = '2.34.1'
+__version__ = '2.35.2'
 warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib')
 
 # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev

ibllib/io/__init__.py

@@ -0,0 +1 @@
+"""Loaders for unprocessed IBL video and task data, and parameter files."""

ibllib/io/extractors/__init__.py

@@ -0,0 +1,4 @@
+"""IBL rig data pre-processing functions.
+
+Extractor classes for loading raw rig data and returning ALF compliant pre-processed data.
+"""

ibllib/io/extractors/training_trials.py

@@ -1,5 +1,6 @@
 import logging
 import numpy as np
+from itertools import accumulate
 from packaging import version
 from one.alf.io import AlfBunch
 
@@ -123,19 +124,16 @@ def _extract(self):
         def get_trial_repeat(trial):
             if 'debias_trial' in trial:
                 return trial['debias_trial']
-            else:
+            elif 'contrast' in trial and isinstance(trial['contrast'], dict):
                 return trial['contrast']['type'] == 'RepeatContrast'
+            else:
+                # For advanced choice world before version 8.19.0 there was no 'debias_trial' field
+                # and no debiasing protocol applied, so simply return False
+                assert self.settings['PYBPOD_PROTOCOL'].startswith('_iblrig_tasks_advancedChoiceWorld')
+                return False
 
-        trial_repeated = np.array(list(map(get_trial_repeat, self.bpod_trials))).astype(int)
-        repNum = trial_repeated.copy()
-        c = 0
-        for i in range(len(trial_repeated)):
-            if trial_repeated[i] == 0:
-                c = 0
-                repNum[i] = 0
-                continue
-            c += 1
-            repNum[i] = c
+        trial_repeated = np.fromiter(map(get_trial_repeat, self.bpod_trials), int)
+        repNum = np.fromiter(accumulate(trial_repeated, lambda x, y: x + y if y else 0), int)
         return repNum
 
 
@@ -163,7 +161,7 @@ class FeedbackTimes(BaseBpodTrialsExtractor):
     **Optional:** saves _ibl_trials.feedback_times.npy
 
     Gets reward  and error state init times vectors,
-    checks if theintersection of nans is empty, then
+    checks if the intersection of nans is empty, then
     merges the 2 vectors.
     """
     save_names = '_ibl_trials.feedback_times.npy'

ibllib/io/extractors/training_wheel.py

@@ -1,3 +1,4 @@
+"""Extractors for the wheel position, velocity, and detected movement."""
 import logging
 from collections.abc import Sized
 
@@ -323,16 +324,35 @@ def extract_wheel_moves(re_ts, re_pos, display=False):
 def extract_first_movement_times(wheel_moves, trials, min_qt=None):
     """
     Extracts the time of the first sufficiently large wheel movement for each trial.
+
     To be counted, the movement must occur between go cue / stim on and before feedback /
     response time.  The movement onset is sometimes just before the cue (occurring in the
     gap between quiescence end and cue start, or during the quiescence period but sub-
-    threshold).  The movement is sufficiently large if it is greater than or equal to THRESH
-    :param wheel_moves: dictionary of detected wheel movement onsets and peak amplitudes for
-    use in extracting each trial's time of first movement.
+    threshold).  The movement is sufficiently large if it is greater than or equal to THRESH.
+
+    :param wheel_moves:
     :param trials: dictionary of trial data
-    :param min_qt: the minimum quiescence period, if None a default is used
-    :return: numpy array of first movement times, bool array indicating whether movement
-    crossed response threshold, and array of indices for wheel_moves arrays
+    :param min_qt:
+    :return: numpy array of
+
+    Parameters
+    ----------
+    wheel_moves : dict
+        Dictionary of detected wheel movement onsets and peak amplitudes for use in extracting each
+        trial's time of first movement.
+    trials : dict
+        Dictionary of trial data.
+    min_qt : float
+        The minimum quiescence period in seconds, if None a default is used.
+
+    Returns
+    -------
+    numpy.array
+        First movement times.
+    numpy.array
+        Bool array indicating whether movement crossed response threshold.
+    numpy.array
+        Indices for wheel_moves arrays.
     """
     THRESH = .1  # peak amp should be at least .1 rad; ~1/3rd of the distance to threshold
     MIN_QT = .2  # default minimum enforced quiescence period
@@ -371,6 +391,8 @@ def extract_first_movement_times(wheel_moves, trials, min_qt=None):
 
 class Wheel(BaseBpodTrialsExtractor):
     """
+    Wheel extractor.
+
     Get wheel data from raw files and converts positions into radians mathematical convention
      (anti-clockwise = +) and timestamps into seconds relative to Bpod clock.
     **Optional:** saves _ibl_wheel.times.npy and _ibl_wheel.position.npy

ibllib/io/extractors/video_motion.py

@@ -42,7 +42,7 @@ def find_nearest(array, value):
 class MotionAlignment:
     roi = {'left': ((800, 1020), (233, 1096)), 'right': ((426, 510), (104, 545)), 'body': ((402, 481), (31, 103))}
 
-    def __init__(self, eid=None, one=None, log=logging.getLogger(__name__), **kwargs):
+    def __init__(self, eid=None, one=None, log=logging.getLogger(__name__), stream=False, **kwargs):
         self.one = one or ONE()
         self.eid = eid
         self.session_path = kwargs.pop('session_path', None) or self.one.eid2path(eid)
@@ -51,7 +51,10 @@ def __init__(self, eid=None, one=None, log=logging.getLogger(__name__), **kwargs
         self.trials = self.wheel = self.camera_times = None
         raw_cam_path = self.session_path.joinpath('raw_video_data')
         camera_path = list(raw_cam_path.glob('_iblrig_*Camera.raw.*'))
-        self.video_paths = {vidio.label_from_path(x): x for x in camera_path}
+        if stream:
+            self.video_paths = vidio.url_from_eid(self.eid)
+        else:
+            self.video_paths = {vidio.label_from_path(x): x for x in camera_path}
         self.data = Bunch()
         self.alignment = Bunch()
 
@@ -107,8 +110,8 @@ def load_data(self, download=False):
         if download:
             self.data.wheel = self.one.load_object(self.eid, 'wheel')
             self.data.trials = self.one.load_object(self.eid, 'trials')
-            cam = self.one.load(self.eid, ['camera.times'], dclass_output=True)
-            self.data.camera_times = {vidio.label_from_path(url): ts for ts, url in zip(cam.data, cam.url)}
+            cam, det = self.one.load_datasets(self.eid, ['*Camera.times*'])
+            self.data.camera_times = {vidio.label_from_path(d['rel_path']): ts for ts, d in zip(cam, det)}
         else:
             alf_path = self.session_path / 'alf'
             wheel_path = next(alf_path.rglob('*wheel.timestamps*')).parent
@@ -329,7 +332,7 @@ def animate(i):
             data['im'].set_data(frame)
 
             mkr = find_nearest(wheel.timestamps[wheel_mask], t_x)
-            data['marker'].set_data(wheel.timestamps[wheel_mask][mkr], wheel.position[wheel_mask][mkr])
+            data['marker'].set_data([wheel.timestamps[wheel_mask][mkr]], [wheel.position[wheel_mask][mkr]])
 
             return data['im'], data['ln'], data['marker']