Merge branch 'release/2.40.0'

Author: oliche

brainbox/io/one.py

@@ -866,13 +866,21 @@ def _get_attributes(dataset_types):
         waveform_attributes = list(set(WAVEFORMS_ATTRIBUTES + waveform_attributes))
         return {'spikes': spike_attributes, 'clusters': cluster_attributes, 'waveforms': waveform_attributes}
 
-    def _get_spike_sorting_collection(self, spike_sorter='pykilosort'):
+    def _get_spike_sorting_collection(self, spike_sorter=None):
         """
         Filters a list or array of collections to get the relevant spike sorting dataset
         if there is a pykilosort, load it
         """
-        collection = next(filter(lambda c: c == f'alf/{self.pname}/{spike_sorter}', self.collections), None)
-        # otherwise, prefers the shortest
+        for sorter in list([spike_sorter, 'iblsorter', 'pykilosort']):
+            if sorter is None:
+                continue
+            if sorter == "":
+                collection = next(filter(lambda c: c == f'alf/{self.pname}', self.collections), None)
+            else:
+                collection = next(filter(lambda c: c == f'alf/{self.pname}/{sorter}', self.collections), None)
+            if collection is not None:
+                return collection
+        # if none is found amongst the defaults, prefers the shortest
         collection = collection or next(iter(sorted(filter(lambda c: f'alf/{self.pname}' in c, self.collections), key=len)), None)
         _logger.debug(f"selecting: {collection} to load amongst candidates: {self.collections}")
         return collection
@@ -982,14 +990,13 @@ def download_raw_waveforms(self, **kwargs):
         """
         _logger.debug(f"loading waveforms from {self.collection}")
         return self.one.load_object(
-            self.eid, "waveforms",
-            attribute=["traces", "templates", "table", "channels"],
+            id=self.eid, obj="waveforms", attribute=["traces", "templates", "table", "channels"],
             collection=self._get_spike_sorting_collection("pykilosort"), download_only=True, **kwargs
         )
 
     def raw_waveforms(self, **kwargs):
         wf_paths = self.download_raw_waveforms(**kwargs)
-        return WaveformsLoader(wf_paths[0].parent, wfs_dtype=np.float16)
+        return WaveformsLoader(wf_paths[0].parent)
 
     def load_channels(self, **kwargs):
         """
@@ -1022,7 +1029,7 @@ def load_channels(self, **kwargs):
             self.histology = 'alf'
         return Bunch(channels)
 
-    def load_spike_sorting(self, spike_sorter='pykilosort', revision=None, enforce_version=True, good_units=False, **kwargs):
+    def load_spike_sorting(self, spike_sorter='iblsorter', revision=None, enforce_version=False, good_units=False, **kwargs):
         """
         Loads spikes, clusters and channels
 

brainbox/io/spikeglx.py

@@ -128,6 +128,7 @@ def __init__(self, pid, one, typ='ap', cache_folder=None, remove_cached=False):
         self.file_chunks = self.one.load_dataset(self.eid, f'*.{typ}.ch', collection=f"*{self.pname}")
         meta_file = self.one.load_dataset(self.eid, f'*.{typ}.meta', collection=f"*{self.pname}")
         cbin_rec = self.one.list_datasets(self.eid, collection=f"*{self.pname}", filename=f'*{typ}.*bin', details=True)
+        cbin_rec.index = cbin_rec.index.map(lambda x: (self.eid, x))
         self.url_cbin = self.one.record2url(cbin_rec)[0]
         with open(self.file_chunks, 'r') as f:
             self.chunks = json.load(f)

ibllib/__init__.py

@@ -2,7 +2,7 @@
 import logging
 import warnings
 
-__version__ = '2.39.1'
+__version__ = '2.40.0'
 warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib')
 
 # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev

ibllib/ephys/sync_probes.py

@@ -47,7 +47,7 @@ def sync(ses_path, **kwargs):
         return version3B(ses_path, **kwargs)
 
 
-def version3A(ses_path, display=True, type='smooth', tol=2.1):
+def version3A(ses_path, display=True, type='smooth', tol=2.1, probe_names=None):
     """
     From a session path with _spikeglx_sync arrays extracted, locate ephys files for 3A and
      outputs one sync.timestamps.probeN.npy file per acquired probe. By convention the reference

ibllib/io/extractors/biased_trials.py

@@ -97,7 +97,7 @@ class TrialsTableBiased(BaseBpodTrialsExtractor):
     save_names = ('_ibl_trials.table.pqt', None, None, '_ibl_wheel.timestamps.npy', '_ibl_wheel.position.npy',
                   '_ibl_wheelMoves.intervals.npy', '_ibl_wheelMoves.peakAmplitude.npy', None, None)
     var_names = ('table', 'stimOff_times', 'stimFreeze_times', 'wheel_timestamps', 'wheel_position', 'wheelMoves_intervals',
-                 'wheelMoves_peakAmplitude', 'peakVelocity_times', 'is_final_movement')
+                 'wheelMoves_peakAmplitude', 'wheelMoves_peakVelocity_times', 'is_final_movement')
 
     def _extract(self, extractor_classes=None, **kwargs):
         extractor_classes = extractor_classes or []
@@ -125,7 +125,7 @@ class TrialsTableEphys(BaseBpodTrialsExtractor):
                   '_ibl_wheelMoves.intervals.npy', '_ibl_wheelMoves.peakAmplitude.npy', None,
                   None, None, None, '_ibl_trials.quiescencePeriod.npy')
     var_names = ('table', 'stimOff_times', 'stimFreeze_times', 'wheel_timestamps', 'wheel_position', 'wheelMoves_intervals',
-                 'wheelMoves_peakAmplitude', 'peakVelocity_times', 'is_final_movement',
+                 'wheelMoves_peakAmplitude', 'wheelMoves_peakVelocity_times', 'is_final_movement',
                  'phase', 'position', 'quiescence')
 
     def _extract(self, extractor_classes=None, **kwargs):
@@ -152,12 +152,12 @@ class BiasedTrials(BaseBpodTrialsExtractor):
     save_names = ('_ibl_trials.goCueTrigger_times.npy', '_ibl_trials.stimOnTrigger_times.npy', None,
                   '_ibl_trials.stimOffTrigger_times.npy', None, None, '_ibl_trials.table.pqt',
                   '_ibl_trials.stimOff_times.npy', None, '_ibl_wheel.timestamps.npy', '_ibl_wheel.position.npy',
-                  '_ibl_wheelMoves.intervals.npy', '_ibl_wheelMoves.peakAmplitude.npy', None, None, '_ibl_trials.included.npy',
-                  None, None, '_ibl_trials.quiescencePeriod.npy')
+                  '_ibl_wheelMoves.intervals.npy', '_ibl_wheelMoves.peakAmplitude.npy', None, None,
+                  '_ibl_trials.included.npy', None, None, '_ibl_trials.quiescencePeriod.npy')
     var_names = ('goCueTrigger_times', 'stimOnTrigger_times', 'itiIn_times', 'stimOffTrigger_times', 'stimFreezeTrigger_times',
                  'errorCueTrigger_times', 'table', 'stimOff_times', 'stimFreeze_times', 'wheel_timestamps', 'wheel_position',
-                 'wheelMoves_intervals', 'wheelMoves_peakAmplitude', 'peakVelocity_times', 'is_final_movement', 'included',
-                 'phase', 'position', 'quiescence')
+                 'wheelMoves_intervals', 'wheelMoves_peakAmplitude', 'wheelMoves_peakVelocity_times', 'is_final_movement',
+                 'included', 'phase', 'position', 'quiescence')
 
     def _extract(self, extractor_classes=None, **kwargs) -> dict:
         extractor_classes = extractor_classes or []
@@ -182,8 +182,8 @@ class EphysTrials(BaseBpodTrialsExtractor):
                   '_ibl_trials.included.npy', None, None, '_ibl_trials.quiescencePeriod.npy')
     var_names = ('goCueTrigger_times', 'stimOnTrigger_times', 'itiIn_times', 'stimOffTrigger_times', 'stimFreezeTrigger_times',
                  'errorCueTrigger_times', 'table', 'stimOff_times', 'stimFreeze_times', 'wheel_timestamps', 'wheel_position',
-                 'wheelMoves_intervals', 'wheelMoves_peakAmplitude', 'peakVelocity_times', 'is_final_movement', 'included',
-                 'phase', 'position', 'quiescence')
+                 'wheelMoves_intervals', 'wheelMoves_peakAmplitude', 'wheelMoves_peakVelocity_times', 'is_final_movement',
+                 'included', 'phase', 'position', 'quiescence')
 
     def _extract(self, extractor_classes=None, **kwargs) -> dict:
         extractor_classes = extractor_classes or []

ibllib/io/extractors/bpod_trials.py

@@ -3,17 +3,14 @@
 This module will extract the Bpod trials and wheel data based on the task protocol,
 i.e. habituation, training or biased.
 """
-import logging
 import importlib
 
-from ibllib.io.extractors.base import get_bpod_extractor_class, protocol2extractor
+from ibllib.io.extractors.base import get_bpod_extractor_class, protocol2extractor, BaseExtractor
 from ibllib.io.extractors.habituation_trials import HabituationTrials
 from ibllib.io.extractors.training_trials import TrainingTrials
 from ibllib.io.extractors.biased_trials import BiasedTrials, EphysTrials
 from ibllib.io.extractors.base import BaseBpodTrialsExtractor
 
-_logger = logging.getLogger(__name__)
-
 
 def get_bpod_extractor(session_path, protocol=None, task_collection='raw_behavior_data') -> BaseBpodTrialsExtractor:
     """
@@ -39,20 +36,25 @@ def get_bpod_extractor(session_path, protocol=None, task_collection='raw_behavio
         'BiasedTrials': BiasedTrials,
         'EphysTrials': EphysTrials
     }
+
     if protocol:
-        class_name = protocol2extractor(protocol)
+        extractor_class_name = protocol2extractor(protocol)
     else:
-        class_name = get_bpod_extractor_class(session_path, task_collection=task_collection)
-    if class_name in builtins:
-        return builtins[class_name](session_path)
+        extractor_class_name = get_bpod_extractor_class(session_path, task_collection=task_collection)
+    if extractor_class_name in builtins:
+        return builtins[extractor_class_name](session_path)
 
     # look if there are custom extractor types in the personal projects repo
-    if not class_name.startswith('projects.'):
-        class_name = 'projects.' + class_name
-    module, class_name = class_name.rsplit('.', 1)
+    if not extractor_class_name.startswith('projects.'):
+        extractor_class_name = 'projects.' + extractor_class_name
+    module, extractor_class_name = extractor_class_name.rsplit('.', 1)
     mdl = importlib.import_module(module)
-    extractor_class = getattr(mdl, class_name, None)
+    extractor_class = getattr(mdl, extractor_class_name, None)
     if extractor_class:
-        return extractor_class(session_path)
+        my_extractor = extractor_class(session_path)
+        if not isinstance(my_extractor, BaseExtractor):
+            raise ValueError(
+                f"{my_extractor} should be an Extractor class inheriting from ibllib.io.extractors.base.BaseExtractor")
+        return my_extractor
     else:
-        raise ValueError(f'extractor {class_name} not found')
+        raise ValueError(f'extractor {extractor_class_name} not found')

ibllib/io/extractors/ephys_fpga.py

@@ -585,13 +585,13 @@ class FpgaTrials(extractors_base.BaseExtractor):
                   '_ibl_trials.stimOff_times.npy', None, None, None, '_ibl_trials.quiescencePeriod.npy',
                   '_ibl_trials.table.pqt', '_ibl_wheel.timestamps.npy',
                   '_ibl_wheel.position.npy', '_ibl_wheelMoves.intervals.npy',
-                  '_ibl_wheelMoves.peakAmplitude.npy')
+                  '_ibl_wheelMoves.peakAmplitude.npy', None)
     var_names = ('goCueTrigger_times', 'stimOnTrigger_times',
                  'stimOffTrigger_times', 'stimFreezeTrigger_times', 'errorCueTrigger_times',
                  'errorCue_times', 'itiIn_times', 'stimFreeze_times', 'stimOff_times',
                  'valveOpen_times', 'phase', 'position', 'quiescence', 'table',
                  'wheel_timestamps', 'wheel_position',
-                 'wheelMoves_intervals', 'wheelMoves_peakAmplitude')
+                 'wheelMoves_intervals', 'wheelMoves_peakAmplitude', 'wheelMoves_peakVelocity_times')
 
     bpod_rsync_fields = ('intervals', 'response_times', 'goCueTrigger_times',
                          'stimOnTrigger_times', 'stimOffTrigger_times',

ibllib/io/extractors/fibrephotometry.py

@@ -225,7 +225,7 @@ def _extract(self, light_source_map=None, collection=None, regions=None, **kwarg
         regions = regions or [k for k in fp_data['raw'].keys() if 'Region' in k]
         out_df = fp_data['raw'].filter(items=regions, axis=1).sort_index(axis=1)
         out_df['times'] = ts
-        out_df['wavelength'] = np.NaN
+        out_df['wavelength'] = np.nan
         out_df['name'] = ''
         out_df['color'] = ''
         # Extract channel index

ibllib/io/extractors/mesoscope.py

@@ -4,8 +4,8 @@
 import numpy as np
 from scipy.signal import find_peaks
 import one.alf.io as alfio
-from one.util import ensure_list
 from one.alf.files import session_path_parts
+from iblutil.util import ensure_list
 import matplotlib.pyplot as plt
 from packaging import version
 

ibllib/io/extractors/opto_trials.py

@@ -16,8 +16,8 @@ class LaserBool(BaseBpodTrialsExtractor):
     def _extract(self, **kwargs):
         _logger.info('Extracting laser datasets')
         # reference pybpod implementation
-        lstim = np.array([float(t.get('laser_stimulation', np.NaN)) for t in self.bpod_trials])
-        lprob = np.array([float(t.get('laser_probability', np.NaN)) for t in self.bpod_trials])
+        lstim = np.array([float(t.get('laser_stimulation', np.nan)) for t in self.bpod_trials])
+        lprob = np.array([float(t.get('laser_probability', np.nan)) for t in self.bpod_trials])
 
         # Karolina's choice world legacy implementation - from Slack message:
         # it is possible that some versions I have used:
@@ -30,9 +30,9 @@ def _extract(self, **kwargs):
         # laserOFF_trials=(optoOUT ==0);
         if 'PROBABILITY_OPTO' in self.settings.keys() and np.all(np.isnan(lstim)):
             lprob = np.zeros_like(lprob) + self.settings['PROBABILITY_OPTO']
-            lstim = np.array([float(t.get('opto_ON_time', np.NaN)) for t in self.bpod_trials])
+            lstim = np.array([float(t.get('opto_ON_time', np.nan)) for t in self.bpod_trials])
             if np.all(np.isnan(lstim)):
-                lstim = np.array([float(t.get('optoOUT', np.NaN)) for t in self.bpod_trials])
+                lstim = np.array([float(t.get('optoOUT', np.nan)) for t in self.bpod_trials])
                 lstim[lstim == 255] = 1
             else:
                 lstim[~np.isnan(lstim)] = 1