Merge branch 'release/2.12.0'

Author: mayofaulkner

.github/workflows/ibllib_ci.yml

@@ -17,8 +17,8 @@ jobs:
       fail-fast: false  # Whether to stop execution of other instances
       max-parallel: 4
       matrix:
-        os: ["ubuntu-latest"] #  "windows-latest"]  #  , "macos-latest"
-        python-version: ["3.8"]  # "3.7",
+        os: ["ubuntu-latest"]
+        python-version: ["3.8"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

MANIFEST.in

@@ -1,6 +1,7 @@
 include ibllib/atlas/allen_structure_tree.csv
 include ibllib/atlas/beryl.npy
 include ibllib/atlas/cosmos.npy
+include ibllib/atlas/swanson.npy
 include ibllib/atlas/mappings.pqt
 include ibllib/io/extractors/extractor_types.json
 include brainbox/tests/wheel_test.p

README.md

@@ -12,9 +12,9 @@ The library is currently 2 main modules:
 [Release Notes here](release_notes.md)
 
 ## Requirements
-**OS**: Deployed on Linux and Windows. Minimally tested for Mac.
+**OS**: Only tested on Linux. Windows and Mac may work, but are not supported.
 
-**Python Module**: Python 3.7 or higher, we develop on 3.8.
+**Python Module**: Python 3.8 or higher
 
 ## Installation, documentation and examples
 https://docs.internationalbrainlab.org
@@ -25,7 +25,7 @@ See https://int-brain-lab.github.io/iblenv/07_contribution.html
 
 We use gitflow and Semantic Versioning.
 
-Before commiting to your branch:
+Before committing to your branch:
 -   run tests
 -   flake8
 This is also enforced by continuous integration.

brainbox/__init__.py

@@ -1,4 +1,5 @@
+import logging
 try:
     import one
 except ModuleNotFoundError:
-    logging.getLogger('ibllib').error('Missing dependency, please run `pip install ONE-api`')
+    logging.getLogger(__name__).error('Missing dependency, please run `pip install ONE-api`')

brainbox/behavior/dlc.py

@@ -11,7 +11,7 @@
 import scipy.interpolate as interpolate
 from scipy.stats import zscore
 
-from ibllib.dsp.smooth import smooth_interpolate_savgol
+from neurodsp.smooth import smooth_interpolate_savgol
 from brainbox.processing import bincount2D
 import brainbox.behavior.wheel as bbox_wheel
 

brainbox/behavior/wheel.py

@@ -213,10 +213,8 @@ def movements(t, pos, freq=1000, pos_thresh=8, t_thresh=.2, min_gap=.1, pos_thre
         if i2proc[-1] == t.size - 1:
             break
 
-    moving = max_disp > pos_thresh  # for each window is the change in position greater than
-    # our threshold?
-    moving = np.insert(moving, 0, False)  # First sample should always be not moving to ensure
-    # we have an onset
+    moving = max_disp > pos_thresh  # for each window is the change in position greater than our threshold?
+    moving = np.insert(moving, 0, False)  # First sample should always be not moving to ensure we have an onset
     moving[-1] = False  # Likewise, ensure we always end on an offset
 
     onset_samps = np.where(~moving[:-1] & moving[1:])[0]

brainbox/io/one.py

@@ -10,12 +10,13 @@
 
 from one.api import ONE, One
 import one.alf.io as alfio
+from one.alf.files import get_alf_path
 from one.alf import cache
+from neuropixel import SITES_COORDINATES, TIP_SIZE_UM, trace_header
+import spikeglx
 
 from iblutil.util import Bunch
-from ibllib.io import spikeglx
 from ibllib.io.extractors.training_wheel import extract_wheel_moves, extract_first_movement_times
-from ibllib.ephys.neuropixel import SITES_COORDINATES, TIP_SIZE_UM, trace_header
 from ibllib.atlas import atlas, AllenAtlas
 from ibllib.pipes import histology
 from ibllib.pipes.ephys_alignment import EphysAlignment
@@ -123,7 +124,7 @@ def _channels_alf2bunch(channels, brain_regions=None):
 
 
 def _load_spike_sorting(eid, one=None, collection=None, revision=None, return_channels=True, dataset_types=None,
-                        brain_regions=None, return_collection=False):
+                        brain_regions=None):
     """
     Generic function to load spike sorting according data using ONE.
 
@@ -168,7 +169,7 @@ def _load_spike_sorting(eid, one=None, collection=None, revision=None, return_ch
     collections = one.list_collections(eid, filename='spikes*', collection=collection, revision=revision)
     if len(collections) == 0:
         _logger.warning(f"eid {eid}: no collection found with collection filter: {collection}, revision: {revision}")
-    pnames = list(set([c.split('/')[1] for c in collections]))
+    pnames = list(set(c.split('/')[1] for c in collections))
     spikes, clusters, channels = ({} for _ in range(3))
 
     spike_attributes, cluster_attributes = _get_attributes(dataset_types)
@@ -246,7 +247,7 @@ def channel_locations_interpolation(channels_aligned, channels=None, brain_regio
     if channels is None:
         channels = {'localCoordinates': np.c_[h['x'], h['y']]}
     nch = channels['localCoordinates'].shape[0]
-    if set(['x', 'y', 'z']).issubset(set(channels_aligned.keys())):
+    if {'x', 'y', 'z'}.issubset(set(channels_aligned.keys())):
         channels_aligned = _channels_bunch2alf(channels_aligned)
     if 'localCoordinates' in channels_aligned.keys():
         aligned_depths = channels_aligned['localCoordinates'][:, 1]
@@ -350,7 +351,7 @@ def _load_channel_locations_traj(eid, probe=None, one=None, revision=None, align
         return channels
 
 
-def load_channel_locations(eid, probe=None, one=None, aligned=False, brain_atlas=None, return_source=False):
+def load_channel_locations(eid, probe=None, one=None, aligned=False, brain_atlas=None):
     """
     Load the brain locations of each channel for a given session/probe
 
@@ -367,8 +368,6 @@ def load_channel_locations(eid, probe=None, one=None, aligned=False, brain_atlas
         Whether to get the latest user aligned channel when not resolved or use histology track
     brain_atlas : ibllib.atlas.BrainAtlas
         Brain atlas object (default: Allen atlas)
-    return_source: bool
-        if True returns the source of the channel lcoations (default False)
     Returns
     -------
     dict of one.alf.io.AlfBunch
@@ -412,7 +411,6 @@ def load_spike_sorting_fast(eid, one=None, probe=None, dataset_types=None, spike
     :param dataset_types: additional spikes/clusters objects to add to the standard default list
     :param spike_sorter: name of the spike sorting you want to load (None for default)
     :param collection: name of the spike sorting collection to load - exclusive with spike sorter name ex: "alf/probe00"
-    :param return_channels: (bool) defaults to False otherwise tries and load channels from disk
     :param brain_regions: ibllib.atlas.regions.BrainRegions object - will label acronyms if provided
     :param nested: if a single probe is required, do not output a dictionary with the probe name as key
     :param return_collection: (False) if True, will return the collection used to load
@@ -454,7 +452,6 @@ def load_spike_sorting(eid, one=None, probe=None, dataset_types=None, spike_sort
     :param probe: name of probe to load in, if not given all probes for session will be loaded
     :param dataset_types: additional spikes/clusters objects to add to the standard default list
     :param spike_sorter: name of the spike sorting you want to load (None for default)
-    :param return_channels: (bool) defaults to False otherwise tries and load channels from disk
     :param brain_regions: ibllib.atlas.regions.BrainRegions object - will label acronyms if provided
     :param return_collection:(bool - False) if True, returns the collection for loading the data
     :return: spikes, clusters (dict of bunch, 1 bunch per probe)
@@ -677,7 +674,7 @@ def load_wheel_reaction_times(eid, one=None):
     eid : [str, UUID, Path, dict]
         Experiment session identifier; may be a UUID, URL, experiment reference string
         details dict or Path
-    one : oneibl.one.OneAlyx, optional
+    one : one.api.OneAlyx, optional
         one object to use for loading. Will generate internal one if not used, by default None
 
     Returns
@@ -703,8 +700,9 @@ def load_wheel_reaction_times(eid, one=None):
     return firstMove_times - trials['goCue_times']
 
 
-def load_trials_df(eid, one=None, maxlen=None, t_before=0., t_after=0., ret_wheel=False,
-                   ret_abswheel=False, wheel_binsize=0.02, addtl_types=[]):
+def load_trials_df(eid, one=None, maxlen=None, t_before=0., t_after=0.2, ret_wheel=False,
+                   ret_abswheel=False, wheel_binsize=0.02, addtl_types=[],
+                   align_event='stimOn_times', keeptrials=None):
     """
     Generate a pandas dataframe of per-trial timing information about a given session.
     Each row in the frame will correspond to a single trial, with timing values indicating timing
@@ -720,7 +718,7 @@ def load_trials_df(eid, one=None, maxlen=None, t_before=0., t_after=0., ret_whee
     eid : [str, UUID, Path, dict]
         Experiment session identifier; may be a UUID, URL, experiment reference string
         details dict or Path
-    one : oneibl.one.OneAlyx, optional
+    one : one.api.OneAlyx, optional
         one object to use for loading. Will generate internal one if not used, by default None
     maxlen : float, optional
         Maximum trial length for inclusion in df. Trials where feedback - response is longer
@@ -779,18 +777,19 @@ def remap_trialp(probs):
     endtimes = trials.feedback_times
     tmp = {key: value for key, value in trials.items() if key in trialstypes}
 
-    if maxlen is not None:
-        with np.errstate(invalid='ignore'):
-            keeptrials = (endtimes - starttimes) <= maxlen
-    else:
-        keeptrials = range(len(starttimes))
+    if keeptrials is None:
+        if maxlen is not None:
+            with np.errstate(invalid='ignore'):
+                keeptrials = (endtimes - starttimes) <= maxlen
+        else:
+            keeptrials = range(len(starttimes))
     trialdata = {x: tmp[x][keeptrials] for x in trialstypes}
     trialdata['probabilityLeft'] = remap_trialp(trialdata['probabilityLeft'])
     trialsdf = pd.DataFrame(trialdata)
     if maxlen is not None:
         trialsdf.set_index(np.nonzero(keeptrials)[0], inplace=True)
-    trialsdf['trial_start'] = trialsdf['stimOn_times'] - t_before
-    trialsdf['trial_end'] = trialsdf['feedback_times'] + t_after
+    trialsdf['trial_start'] = trialsdf[align_event] - t_before
+    trialsdf['trial_end'] = trialsdf[align_event] + t_after
     tdiffs = trialsdf['trial_end'] - np.roll(trialsdf['trial_start'], -1)
     if np.any(tdiffs[:-1] > 0):
         logging.warning(f'{sum(tdiffs[:-1] > 0)} trials overlapping due to t_before and t_after '
@@ -879,16 +878,17 @@ class SpikeSortingLoader:
             SpikeSortingLoader(eid=eid, pname='probe00', one=one)
     - From a local session and probe name:
             SpikeSortingLoader(session_path=session_path, pname='probe00')
+    NB: When no ONE instance is passed, any datasets that are loaded will not be recorded.
     """
-    one: ONE = None
+    one: One = None
     atlas: None = None
     pid: str = None
     eid: str = ''
     pname: str = ''
-    # the following properties are the outcome of the post init funciton
     session_path: Path = ''
+    # the following properties are the outcome of the post init function
     collections: list = None
-    datasets: list = None   # list of all datasets belonging to the sesion
+    datasets: list = None   # list of all datasets belonging to the session
     # the following properties are the outcome of a reading function
     files: dict = None
     collection: str = ''
@@ -905,11 +905,14 @@ def __post_init__(self):
             self.session_path = self.one.eid2path(self.eid)
         # fully local providing a session path
         else:
-            self.one = One(cache_dir=self.session_path.parents[2], mode='local')
-            df_sessions = cache._make_sessions_df(self.session_path)
-            self.one._cache['sessions'] = df_sessions.set_index('id')
-            self.one._cache['datasets'] = cache._make_datasets_df(self.session_path, hash_files=False)
-            self.eid = str(self.session_path.relative_to(self.session_path.parents[2]))
+            if self.one:
+                self.eid = self.one.to_eid(self.session_path)
+            else:
+                self.one = One(cache_dir=self.session_path.parents[2], mode='local')
+                df_sessions = cache._make_sessions_df(self.session_path)
+                self.one._cache['sessions'] = df_sessions.set_index('id')
+                self.one._cache['datasets'] = cache._make_datasets_df(self.session_path, hash_files=False)
+                self.eid = str(self.session_path.relative_to(self.session_path.parents[2]))
         # populates default properties
         self.collections = self.one.list_collections(
             self.eid, filename='spikes*', collection=f"alf/{self.pname}*")
@@ -930,7 +933,7 @@ def _get_attributes(dataset_types):
             cluster_attributes = list(set(CLUSTERS_ATTRIBUTES + cluster_attributes))
             return spike_attributes, cluster_attributes
 
-    def _get_spike_sorting_collection(self, spike_sorter='pykilosort', revision=None):
+    def _get_spike_sorting_collection(self, spike_sorter='pykilosort'):
         """
         Filters a list or array of collections to get the relevant spike sorting dataset
         if there is a pykilosort, load it
@@ -982,7 +985,7 @@ def load_spike_sorting(self, **kwargs):
         -   alf: the final version of channel locations, same as resolved with the difference that data is on file
         -   resolved: channel locations alignments have been agreed upon
         -   aligned: channel locations have been aligned, but review or other alignments are pending, potentially not accurate
-        -   traced: the histology track has been recovered from microscopy, however the depths may not match, inacurate data
+        -   traced: the histology track has been recovered from microscopy, however the depths may not match, inaccurate data
 
         :param spike_sorter: (defaults to 'pykilosort')
         :param dataset_types: list of extra dataset types
@@ -1034,4 +1037,5 @@ def merge_clusters(spikes, clusters, channels, cache_dir=None):
     @property
     def url(self):
         """Gets flatiron URL for the session"""
-        return str(self.session_path).replace(str(self.one.alyx.cache_dir), 'https://ibl.flatironinstitute.org')
+        webclient = getattr(self.one, '_web_client', None)
+        return webclient.rel_path2url(get_alf_path(self.session_path)) if webclient else None