Merge branch 'release/2.12.0' into mtnn

Author: mayofaulkner

.github/workflows/ibllib_ci.yml

@@ -17,8 +17,8 @@ jobs:
       fail-fast: false  # Whether to stop execution of other instances
       max-parallel: 4
       matrix:
-        os: ["ubuntu-latest"] #  "windows-latest"]  #  , "macos-latest"
-        python-version: ["3.8"]  # "3.7",
+        os: ["ubuntu-latest"]
+        python-version: ["3.8"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

MANIFEST.in

@@ -1,6 +1,7 @@
 include ibllib/atlas/allen_structure_tree.csv
 include ibllib/atlas/beryl.npy
 include ibllib/atlas/cosmos.npy
+include ibllib/atlas/swanson.npy
 include ibllib/atlas/mappings.pqt
 include ibllib/io/extractors/extractor_types.json
 include brainbox/tests/wheel_test.p

README.md

@@ -12,9 +12,9 @@ The library is currently 2 main modules:
 [Release Notes here](release_notes.md)
 
 ## Requirements
-**OS**: Deployed on Linux and Windows. Minimally tested for Mac.
+**OS**: Only tested on Linux. Windows and Mac may work, but are not supported.
 
-**Python Module**: Python 3.7 or higher, we develop on 3.8.
+**Python Module**: Python 3.8 or higher
 
 ## Installation, documentation and examples
 https://docs.internationalbrainlab.org
@@ -25,7 +25,7 @@ See https://int-brain-lab.github.io/iblenv/07_contribution.html
 
 We use gitflow and Semantic Versioning.
 
-Before commiting to your branch:
+Before committing to your branch:
 -   run tests
 -   flake8
 This is also enforced by continuous integration.

brainbox/__init__.py

@@ -1,4 +1,5 @@
+import logging
 try:
     import one
 except ModuleNotFoundError:
-    logging.getLogger('ibllib').error('Missing dependency, please run `pip install ONE-api`')
+    logging.getLogger(__name__).error('Missing dependency, please run `pip install ONE-api`')

brainbox/behavior/dlc.py

@@ -11,7 +11,7 @@
 import scipy.interpolate as interpolate
 from scipy.stats import zscore
 
-from ibllib.dsp.smooth import smooth_interpolate_savgol
+from neurodsp.smooth import smooth_interpolate_savgol
 from brainbox.processing import bincount2D
 import brainbox.behavior.wheel as bbox_wheel
 
@@ -458,16 +458,17 @@ def plot_motion_energy_hist(camera_dict, trials_df):
                 motion_energy = zscore(camera_dict[cam]['motion_energy'], nan_policy='omit')
                 try:
                     start_idx = insert_idx(camera_dict[cam]['times'], start_window)
+                    end_idx = np.array(start_idx + int(WINDOW_LEN * SAMPLING[cam]), dtype='int64')
+                    me_all = [motion_energy[start_idx[i]:end_idx[i]] for i in range(len(start_idx))]
+                    me_all = [m for m in me_all if len(m) > 0]
+                    times = np.arange(len(me_all[0])) / SAMPLING[cam] + WINDOW_LAG
+                    me_mean = np.mean(me_all, axis=0)
+                    me_std = np.std(me_all, axis=0) / np.sqrt(len(me_all))
+                    plt.plot(times, me_mean, label=f'{cam} cam', color=colors[cam], linewidth=2)
+                    plt.fill_between(times, me_mean + me_std, me_mean - me_std, color=colors[cam], alpha=0.2)
                 except ValueError:
-                    logger.error("Camera.times are outside of the trial windows")
-                    raise
-                end_idx = np.array(start_idx + int(WINDOW_LEN * SAMPLING[cam]), dtype='int64')
-                me_all = [motion_energy[start_idx[i]:end_idx[i]] for i in range(len(start_idx))]
-                times = np.arange(len(me_all[0])) / SAMPLING[cam] + WINDOW_LAG
-                me_mean = np.mean(me_all, axis=0)
-                me_std = np.std(me_all, axis=0) / np.sqrt(len(me_all))
-                plt.plot(times, me_mean, label=f'{cam} cam', color=colors[cam], linewidth=2)
-                plt.fill_between(times, me_mean + me_std, me_mean - me_std, color=colors[cam], alpha=0.2)
+                    logger.error(f"{cam}Camera camera.times are outside of the trial windows")
+                    missing_data.append(cam)
             except AttributeError:
                 logger.warning(f"Cannot load motion energy and/or times data for {cam} camera")
                 missing_data.append(cam)
@@ -484,7 +485,7 @@ def plot_motion_energy_hist(camera_dict, trials_df):
     if len(missing_data) > 0:
         ax = plt.gca()
         ax.text(.95, .35, f"Data incomplete for\n{' and '.join(missing_data)} camera", color='r', fontsize=10,
-                horizontalalignment='right', verticalalignment='center', transform=ax.transAxes)
+                fontweight='bold', horizontalalignment='right', verticalalignment='center', transform=ax.transAxes)
     return plt.gca()
 
 

brainbox/behavior/wheel.py

@@ -213,10 +213,8 @@ def movements(t, pos, freq=1000, pos_thresh=8, t_thresh=.2, min_gap=.1, pos_thre
         if i2proc[-1] == t.size - 1:
             break
 
-    moving = max_disp > pos_thresh  # for each window is the change in position greater than
-    # our threshold?
-    moving = np.insert(moving, 0, False)  # First sample should always be not moving to ensure
-    # we have an onset
+    moving = max_disp > pos_thresh  # for each window is the change in position greater than our threshold?
+    moving = np.insert(moving, 0, False)  # First sample should always be not moving to ensure we have an onset
     moving[-1] = False  # Likewise, ensure we always end on an offset
 
     onset_samps = np.where(~moving[:-1] & moving[1:])[0]

brainbox/io/one.py

@@ -10,12 +10,13 @@
 
 from one.api import ONE, One
 import one.alf.io as alfio
+from one.alf.files import get_alf_path
 from one.alf import cache
+from neuropixel import SITES_COORDINATES, TIP_SIZE_UM, trace_header
+import spikeglx
 
 from iblutil.util import Bunch
-from ibllib.io import spikeglx
 from ibllib.io.extractors.training_wheel import extract_wheel_moves, extract_first_movement_times
-from ibllib.ephys.neuropixel import SITES_COORDINATES, TIP_SIZE_UM, trace_header
 from ibllib.atlas import atlas, AllenAtlas
 from ibllib.pipes import histology
 from ibllib.pipes.ephys_alignment import EphysAlignment
@@ -123,7 +124,7 @@ def _channels_alf2bunch(channels, brain_regions=None):
 
 
 def _load_spike_sorting(eid, one=None, collection=None, revision=None, return_channels=True, dataset_types=None,
-                        brain_regions=None, return_collection=False):
+                        brain_regions=None):
     """
     Generic function to load spike sorting according data using ONE.
 
@@ -168,7 +169,7 @@ def _load_spike_sorting(eid, one=None, collection=None, revision=None, return_ch
     collections = one.list_collections(eid, filename='spikes*', collection=collection, revision=revision)
     if len(collections) == 0:
         _logger.warning(f"eid {eid}: no collection found with collection filter: {collection}, revision: {revision}")
-    pnames = list(set([c.split('/')[1] for c in collections]))
+    pnames = list(set(c.split('/')[1] for c in collections))
     spikes, clusters, channels = ({} for _ in range(3))
 
     spike_attributes, cluster_attributes = _get_attributes(dataset_types)
@@ -246,7 +247,7 @@ def channel_locations_interpolation(channels_aligned, channels=None, brain_regio
     if channels is None:
         channels = {'localCoordinates': np.c_[h['x'], h['y']]}
     nch = channels['localCoordinates'].shape[0]
-    if set(['x', 'y', 'z']).issubset(set(channels_aligned.keys())):
+    if {'x', 'y', 'z'}.issubset(set(channels_aligned.keys())):
         channels_aligned = _channels_bunch2alf(channels_aligned)
     if 'localCoordinates' in channels_aligned.keys():
         aligned_depths = channels_aligned['localCoordinates'][:, 1]
@@ -350,7 +351,7 @@ def _load_channel_locations_traj(eid, probe=None, one=None, revision=None, align
         return channels
 
 
-def load_channel_locations(eid, probe=None, one=None, aligned=False, brain_atlas=None, return_source=False):
+def load_channel_locations(eid, probe=None, one=None, aligned=False, brain_atlas=None):
     """
     Load the brain locations of each channel for a given session/probe
 
@@ -367,8 +368,6 @@ def load_channel_locations(eid, probe=None, one=None, aligned=False, brain_atlas
         Whether to get the latest user aligned channel when not resolved or use histology track
     brain_atlas : ibllib.atlas.BrainAtlas
         Brain atlas object (default: Allen atlas)
-    return_source: bool
-        if True returns the source of the channel lcoations (default False)
     Returns
     -------
     dict of one.alf.io.AlfBunch
@@ -412,7 +411,6 @@ def load_spike_sorting_fast(eid, one=None, probe=None, dataset_types=None, spike
     :param dataset_types: additional spikes/clusters objects to add to the standard default list
     :param spike_sorter: name of the spike sorting you want to load (None for default)
     :param collection: name of the spike sorting collection to load - exclusive with spike sorter name ex: "alf/probe00"
-    :param return_channels: (bool) defaults to False otherwise tries and load channels from disk
     :param brain_regions: ibllib.atlas.regions.BrainRegions object - will label acronyms if provided
     :param nested: if a single probe is required, do not output a dictionary with the probe name as key
     :param return_collection: (False) if True, will return the collection used to load
@@ -454,7 +452,6 @@ def load_spike_sorting(eid, one=None, probe=None, dataset_types=None, spike_sort
     :param probe: name of probe to load in, if not given all probes for session will be loaded
     :param dataset_types: additional spikes/clusters objects to add to the standard default list
     :param spike_sorter: name of the spike sorting you want to load (None for default)
-    :param return_channels: (bool) defaults to False otherwise tries and load channels from disk
     :param brain_regions: ibllib.atlas.regions.BrainRegions object - will label acronyms if provided
     :param return_collection:(bool - False) if True, returns the collection for loading the data
     :return: spikes, clusters (dict of bunch, 1 bunch per probe)
@@ -677,7 +674,7 @@ def load_wheel_reaction_times(eid, one=None):
     eid : [str, UUID, Path, dict]
         Experiment session identifier; may be a UUID, URL, experiment reference string
         details dict or Path
-    one : oneibl.one.OneAlyx, optional
+    one : one.api.OneAlyx, optional
         one object to use for loading. Will generate internal one if not used, by default None
 
     Returns
@@ -721,7 +718,7 @@ def load_trials_df(eid, one=None, maxlen=None, t_before=0., t_after=0., ret_whee
     eid : [str, UUID, Path, dict]
         Experiment session identifier; may be a UUID, URL, experiment reference string
         details dict or Path
-    one : oneibl.one.OneAlyx, optional
+    one : one.api.OneAlyx, optional
         one object to use for loading. Will generate internal one if not used, by default None
     maxlen : float, optional
         Maximum trial length for inclusion in df. Trials where feedback - response is longer
@@ -881,16 +878,17 @@ class SpikeSortingLoader:
             SpikeSortingLoader(eid=eid, pname='probe00', one=one)
     - From a local session and probe name:
             SpikeSortingLoader(session_path=session_path, pname='probe00')
+    NB: When no ONE instance is passed, any datasets that are loaded will not be recorded.
     """
-    one: ONE = None
+    one: One = None
     atlas: None = None
     pid: str = None
     eid: str = ''
     pname: str = ''
-    # the following properties are the outcome of the post init funciton
     session_path: Path = ''
+    # the following properties are the outcome of the post init function
     collections: list = None
-    datasets: list = None   # list of all datasets belonging to the sesion
+    datasets: list = None   # list of all datasets belonging to the session
     # the following properties are the outcome of a reading function
     files: dict = None
     collection: str = ''
@@ -907,11 +905,14 @@ def __post_init__(self):
             self.session_path = self.one.eid2path(self.eid)
         # fully local providing a session path
         else:
-            self.one = One(cache_dir=self.session_path.parents[2], mode='local')
-            df_sessions = cache._make_sessions_df(self.session_path)
-            self.one._cache['sessions'] = df_sessions.set_index('id')
-            self.one._cache['datasets'] = cache._make_datasets_df(self.session_path, hash_files=False)
-            self.eid = str(self.session_path.relative_to(self.session_path.parents[2]))
+            if self.one:
+                self.eid = self.one.to_eid(self.session_path)
+            else:
+                self.one = One(cache_dir=self.session_path.parents[2], mode='local')
+                df_sessions = cache._make_sessions_df(self.session_path)
+                self.one._cache['sessions'] = df_sessions.set_index('id')
+                self.one._cache['datasets'] = cache._make_datasets_df(self.session_path, hash_files=False)
+                self.eid = str(self.session_path.relative_to(self.session_path.parents[2]))
         # populates default properties
         self.collections = self.one.list_collections(
             self.eid, filename='spikes*', collection=f"alf/{self.pname}*")
@@ -932,7 +933,7 @@ def _get_attributes(dataset_types):
             cluster_attributes = list(set(CLUSTERS_ATTRIBUTES + cluster_attributes))
             return spike_attributes, cluster_attributes
 
-    def _get_spike_sorting_collection(self, spike_sorter='pykilosort', revision=None):
+    def _get_spike_sorting_collection(self, spike_sorter='pykilosort'):
         """
         Filters a list or array of collections to get the relevant spike sorting dataset
         if there is a pykilosort, load it
@@ -943,17 +944,18 @@ def _get_spike_sorting_collection(self, spike_sorter='pykilosort', revision=None
         _logger.debug(f"selecting: {collection} to load amongst candidates: {self.collections}")
         return collection
 
-    def download_spike_sorting_object(self, obj, spike_sorter='pykilosort', dataset_types=None):
+    def download_spike_sorting_object(self, obj, spike_sorter='pykilosort', dataset_types=None, collection=None):
         """
         Downloads an ALF object
         :param obj: object name, str between 'spikes', 'clusters' or 'channels'
         :param spike_sorter: (defaults to 'pykilosort')
-        :param dataset_types: list of extra dataset types
+        :param dataset_types: list of extra dataset types, for example ['spikes.samples']
+        :param collection: string specifiying the collection, for example 'alf/probe01/pykilosort'
         :return:
         """
         if len(self.collections) == 0:
             return {}, {}, {}
-        self.collection = self._get_spike_sorting_collection(spike_sorter=spike_sorter)
+        self.collection = collection or self._get_spike_sorting_collection(spike_sorter=spike_sorter)
         _logger.debug(f"loading spike sorting from {self.collection}")
         spike_attributes, cluster_attributes = self._get_attributes(dataset_types)
         attributes = {'spikes': spike_attributes, 'clusters': cluster_attributes, 'channels': None,
@@ -983,7 +985,7 @@ def load_spike_sorting(self, **kwargs):
         -   alf: the final version of channel locations, same as resolved with the difference that data is on file
         -   resolved: channel locations alignments have been agreed upon
         -   aligned: channel locations have been aligned, but review or other alignments are pending, potentially not accurate
-        -   traced: the histology track has been recovered from microscopy, however the depths may not match, inacurate data
+        -   traced: the histology track has been recovered from microscopy, however the depths may not match, inaccurate data
 
         :param spike_sorter: (defaults to 'pykilosort')
         :param dataset_types: list of extra dataset types
@@ -1035,4 +1037,5 @@ def merge_clusters(spikes, clusters, channels, cache_dir=None):
     @property
     def url(self):
         """Gets flatiron URL for the session"""
-        return str(self.session_path).replace(str(self.one.alyx.cache_dir), 'https://ibl.flatironinstitute.org')
+        webclient = getattr(self.one, '_web_client', None)
+        return webclient.rel_path2url(get_alf_path(self.session_path)) if webclient else None