Merge branch 'develop' into hotfix/3.3.1

Author: oliche

brainbox/io/one.py

@@ -5,14 +5,15 @@
 import re
 import os
 from pathlib import Path
+from collections import defaultdict
 
 import numpy as np
 import pandas as pd
 from scipy.interpolate import interp1d
 import matplotlib.pyplot as plt
 
 from one.api import ONE, One
-from one.alf.path import get_alf_path, full_path_parts
+from one.alf.path import get_alf_path, full_path_parts, filename_parts
 from one.alf.exceptions import ALFObjectNotFound, ALFMultipleCollectionsFound
 from one.alf import cache
 import one.alf.io as alfio
@@ -193,9 +194,9 @@ def _load_spike_sorting(eid, one=None, collection=None, revision=None, return_ch
     for pname in pnames:
         probe_collection = _get_spike_sorting_collection(collections, pname)
         spikes[pname] = one.load_object(eid, collection=probe_collection, obj='spikes',
-                                        attribute=spike_attributes)
+                                        attribute=spike_attributes, namespace='')
         clusters[pname] = one.load_object(eid, collection=probe_collection, obj='clusters',
-                                          attribute=cluster_attributes)
+                                          attribute=cluster_attributes, namespace='')
     if return_channels:
         channels = _load_channels_locations_from_disk(
             eid, collection=collection, one=one, revision=revision, brain_regions=brain_regions)
@@ -1035,7 +1036,31 @@ def load_channels(self, **kwargs):
             self.histology = 'alf'
         return Bunch(channels)
 
-    def load_spike_sorting(self, spike_sorter='iblsorter', revision=None, enforce_version=False, good_units=False, **kwargs):
+    @staticmethod
+    def filter_files_by_namespace(all_files, namespace):
+
+        # Create dict for each file with available namespaces, no namespce is stored under the key None
+        namespace_files = defaultdict(dict)
+        available_namespaces = []
+        for file in all_files:
+            fparts = filename_parts(file.name, as_dict=True)
+            fname = f"{fparts['object']}.{fparts['attribute']}"
+            nspace = fparts['namespace']
+            available_namespaces.append(nspace)
+            namespace_files[fname][nspace] = file
+
+        if namespace not in set(available_namespaces):
+            _logger.info(f'Could not find manual curation results for {namespace}, returning default'
+                         f' non manually curated spikesorting data')
+
+        # Return the files with the chosen namespace.
+        files = [f.get(namespace, f.get(None, None)) for f in namespace_files.values()]
+        # remove any None files
+        files = [f for f in files if f]
+        return files
+
+    def load_spike_sorting(self, spike_sorter='iblsorter', revision=None, enforce_version=False, good_units=False,
+                           namespace=None, **kwargs):
         """
         Loads spikes, clusters and channels
 
@@ -1053,6 +1078,8 @@ def load_spike_sorting(self, spike_sorter='iblsorter', revision=None, enforce_ve
         :param enforce_version: if True, will raise an error if the spike sorting version and revision is not the expected one
         :param dataset_types: list of extra dataset types, for example: ['spikes.samples', 'spikes.templates']
         :param good_units: False, if True will load only the good units, possibly by downloading a smaller spikes table
+        :param namespace: None, if given will load the manually curated spikesorting with the given namespace,
+                         e.g to load '_av_.clusters.depths use namespace='av'
         :param kwargs: additional arguments to be passed to one.api.One.load_object
         :return:
         """
@@ -1061,13 +1088,21 @@ def load_spike_sorting(self, spike_sorter='iblsorter', revision=None, enforce_ve
         self.files = {}
         self.spike_sorter = spike_sorter
         self.revision = revision
+
+        if good_units and namespace is not None:
+            _logger.info('Good units table does not exist for manually curated spike sorting. Pass in namespace with'
+                         'good_units=False and filter the spikes post hoc by the good clusters.')
+            return [None] * 3
         objects = ['passingSpikes', 'clusters', 'channels'] if good_units else None
         self.download_spike_sorting(spike_sorter=spike_sorter, revision=revision, objects=objects, **kwargs)
         channels = self.load_channels(spike_sorter=spike_sorter, revision=revision, **kwargs)
+        self.files['clusters'] = self.filter_files_by_namespace(self.files['clusters'], namespace)
         clusters = self._load_object(self.files['clusters'], wildcards=self.one.wildcards)
+
         if good_units:
             spikes = self._load_object(self.files['passingSpikes'], wildcards=self.one.wildcards)
         else:
+            self.files['spikes'] = self.filter_files_by_namespace(self.files['spikes'], namespace)
             spikes = self._load_object(self.files['spikes'], wildcards=self.one.wildcards)
         if enforce_version:
             self._assert_version_consistency()

examples/exploring_data/data_download.ipynb

@@ -142,16 +142,18 @@
    ]
   },
   {
-   "metadata": {},
    "cell_type": "markdown",
+   "metadata": {},
    "source": [
     "### Find recordings of a specific brain region\n",
     "If we are interested in a given brain region, we can use the `search_insertions` method to find all recordings associated with that region. For example, to find all recordings associated with the **Rhomboid Nucleus (RH)** region of the thalamus."
    ]
   },
   {
-   "metadata": {},
    "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
     "# this is the query that yields the few recordings for the Rhomboid Nucleus (RH) region\n",
     "insertions_rh = one.search_insertions(atlas_acronym='RH', datasets='spikes.times.npy', project='brainwide')\n",
@@ -161,9 +163,7 @@
     "\n",
     "# the Allen brain regions parcellation is hierarchical, and searching for Thalamus will return all child Rhomboid Nucleus (RH) regions\n",
     "assert set(insertions_rh).issubset(set(insertions_th))\n"
-   ],
-   "outputs": [],
-   "execution_count": null
+   ]
   },
   {
    "cell_type": "markdown",
@@ -183,7 +183,7 @@
    "outputs": [],
    "source": [
     "# Find sessions that have spikes.times datasets\n",
-    "sessions_with_spikes = one.search(project='brainwide', dataset='spikes.times')"
+    "sessions_with_spikes = one.search(project='brainwide', datasets='spikes.times.npy')"
    ]
   },
   {
@@ -253,7 +253,7 @@
    "outputs": [],
    "source": [
     "# Find an example session with trials data\n",
-    "eid, *_ = one.search(project='brainwide', dataset='_ibl_trials.table.pqt')\n",
+    "eid, *_ = one.search(project='brainwide', datasets='_ibl_trials.table.pqt')\n",
     "# List datasets associated with a session, in the alf collection\n",
     "datasets = one.list_datasets(eid, collection='alf*')\n",
     "\n",
@@ -279,7 +279,7 @@
    "source": [
     "# Find an example session with spike data\n",
     "# Note: Restricting by task and project makes searching for data much quicker\n",
-    "eid, *_ = one.search(project='brainwide', dataset='spikes', task='ephys')\n",
+    "eid, *_ = one.search(project='brainwide', datasets='spikes.times.npy', task='ephys')\n",
     "\n",
     "# Data for each probe insertion are stored in the alf/probeXX folder.\n",
     "datasets = one.list_datasets(eid, collection='alf/probe*')\n",
@@ -375,7 +375,7 @@
     "lab_name = list(labs)[0]\n",
     "\n",
     "# Searching for RS sessions with specific lab name\n",
-    "sessions_lab = one.search(dataset='spikes', lab=lab_name)"
+    "sessions_lab = one.search(datasets='spikes', lab=lab_name)"
    ]
   },
   {

examples/loading_data/loading_photometry_data.ipynb

@@ -61,7 +61,7 @@
    "source": [
     "from one.api import ONE\n",
     "one = ONE()\n",
-    "sessions = one.search(dataset='photometry.signal.pqt')\n",
+    "sessions = one.search(datasets='photometry.signal.pqt')\n",
     "print(f'{len(sessions)} sessions with photometry data found')"
    ]
   },
@@ -271,7 +271,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.11.9"
   }
  },
  "nbformat": 4,

examples/loading_data/loading_trials_data.ipynb

@@ -50,7 +50,10 @@
    "cell_type": "markdown",
    "id": "a5d358e035a91310",
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
    },
    "source": [
     "## Loading a single session's trials\n"
@@ -77,7 +80,10 @@
    "cell_type": "markdown",
    "id": "d6c98a81f5426445",
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
    },
    "source": [
     "For combining trials data with various recording modalities for a given session, the `SessionLoader` class is more convenient:"
@@ -130,8 +136,12 @@
     "from one.api import ONE\n",
     "one = ONE()\n",
     "subject = 'SWC_043'\n",
+    "# Load in subject trials table\n",
     "trials = one.load_aggregate('subjects', subject, '_ibl_subjectTrials.table')\n",
     "\n",
+    "# Load in subject sessions table\n",
+    "sessions = one.load_aggregate('subjects', subject, '_ibl_subjectSessions.table')\n",
+    "\n",
     "# Load training status and join to trials table\n",
     "training = one.load_aggregate('subjects', subject, '_ibl_subjectTraining.table')\n",
     "trials = (trials\n",
@@ -141,10 +151,9 @@
     "trials['training_status'] = trials.training_status.fillna(method='ffill')\n",
     "\n",
     "# Join sessions table for number, task_protocol, etc.\n",
-    "trials = one.load_aggregate('subjects', subject, '_ibl_subjectTrials.table')\n",
     "if 'task_protocol' in trials:\n",
-    "    trials.drop('task_protocol', axis=1)\n",
-    "trials = trials.set_index('session').join(one._cache.sessions.drop('date', axis=1))"
+    "    trials = trials.drop('task_protocol', axis=1)\n",
+    "trials = trials.join(sessions.drop('date', axis=1))"
    ]
   },
   {
@@ -302,7 +311,10 @@
    "cell_type": "markdown",
    "id": "55ad2e5d71ac301",
    "metadata": {
-    "collapsed": false
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
    },
    "source": [
     "### Example 5: Computing the inter-trial interval (ITI)\n",
@@ -345,9 +357,9 @@
  "metadata": {
   "celltoolbar": "Edit Metadata",
   "kernelspec": {
-   "display_name": "Python [conda env:iblenv] *",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "name": "conda-env-iblenv-py"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -359,7 +371,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "version": "3.11.9"
   }
  },
  "nbformat": 4,

examples/loading_data/loading_widefield_data.ipynb

@@ -86,7 +86,7 @@
    "source": [
     "from one.api import ONE\n",
     "one = ONE()\n",
-    "sessions = one.search(dataset='widefieldU.images.npy')\n",
+    "sessions = one.search(datasets='widefieldU.images.npy')\n",
     "print(f'{len(sessions)} sessions with widefield data found')"
    ]
   },
@@ -224,7 +224,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.11.9"
   }
  },
  "nbformat": 4,

ibllib/io/extractors/default_channel_maps.py

@@ -62,7 +62,8 @@
                       'audio': 4,
                       'bpod': 5,
                       'rotary_encoder': 6,
-                      'neural_frames': 7}
+                      'neural_frames': 7,
+                      'volume_counter': 8}
          }
 }
 

ibllib/io/extractors/ephys_fpga.py

@@ -35,7 +35,6 @@
 from pathlib import Path
 import uuid
 import re
-from functools import partial
 
 import matplotlib.pyplot as plt
 from matplotlib.colors import TABLEAU_COLORS
@@ -794,6 +793,7 @@ def _extract(self, sync=None, chmap=None, sync_collection='raw_ephys_data',
         elif (protocol_number := kwargs.get('protocol_number')) is not None:  # look for spacer
             # The spacers are TTLs generated by Bpod at the start of each protocol
             tmin, tmax = get_protocol_period(self.session_path, protocol_number, bpod)
+            tmin += (Spacer().times[-1] + Spacer().tup + 0.05)  # exclude spacer itself
         else:
             # Older sessions don't have protocol spacers so we sync the Bpod intervals here to
             # find the approximate end time of the protocol (this will exclude the passive signals
@@ -1453,16 +1453,23 @@ def get_bpod_event_times(self, sync, chmap, bpod_event_ttls=None, display=False,
         # lengths are defined by the state machine of the task protocol and therefore vary.
         if bpod_event_ttls is None:
             # Currently (at least v8.12 and below) there is no trial start or end TTL, only an ITI pulse
-            bpod_event_ttls = {'trial_iti': (1, 1.1), 'valve_open': (0, 0.4)}
+            bpod_event_ttls = {'trial_iti': (.999, 1.1), 'valve_open': (0, 0.4)}
         bpod_event_intervals = self._assign_events(
             bpod['times'], bpod['polarities'], bpod_event_ttls, display=display)
 
         # The first trial pulse is shorter and assigned to valve_open. Here we remove the first
         # valve event, prepend a 0 to the trial_start events, and drop the last trial if it was
         # incomplete in Bpod.
-        bpod_event_intervals['trial_iti'] = np.r_[bpod_event_intervals['valve_open'][0:1, :],
-                                                  bpod_event_intervals['trial_iti']]
-        bpod_event_intervals['valve_open'] = bpod_event_intervals['valve_open'][1:, :]
+        t0 = bpod_event_intervals['trial_iti'][0, 0]  # expect 1st event to be trial_start
+        pretrial = [(k, v[0, 0]) for k, v in bpod_event_intervals.items() if v.size and v[0, 0] < t0]
+        if pretrial:
+            (pretrial, _) = sorted(pretrial, key=lambda x: x[1])[0]  # take the earliest event
+            dt = np.diff(bpod_event_intervals[pretrial][0, :]) * 1e3  # record TTL length to log
+            _logger.debug('Reassigning first %s to trial_start. TTL length = %.3g ms', pretrial, dt)
+            bpod_event_intervals['trial_iti'] = np.r_[
+                bpod_event_intervals[pretrial][0:1, :], bpod_event_intervals['trial_iti']
+            ]
+            bpod_event_intervals[pretrial] = bpod_event_intervals[pretrial][1:, :]
 
         return bpod, bpod_event_intervals
 
@@ -1514,13 +1521,16 @@ def build_trials(self, sync, chmap, display=False, **kwargs):
         out.update({k: self.bpod2fpga(self.bpod_trials[k][ibpod]) for k in self.bpod_rsync_fields})
 
         # Assigning each event to a trial ensures exactly one event per trial (missing events are NaN)
-        assign_to_trial = partial(_assign_events_to_trial, fpga_events['intervals_0'])
         trials = alfio.AlfBunch({
-            'goCue_times': assign_to_trial(fpga_events['goCue_times'], take='first'),
-            'feedback_times': assign_to_trial(fpga_events['feedback_times']),
-            'stimCenter_times': assign_to_trial(self.frame2ttl['times'], take=-2),
-            'stimOn_times': assign_to_trial(self.frame2ttl['times'], take='first'),
-            'stimOff_times': assign_to_trial(self.frame2ttl['times']),
+            'goCue_times': _assign_events_to_trial(out['goCueTrigger_times'], fpga_events['goCue_times'], take='first'),
+            'feedback_times': _assign_events_to_trial(fpga_events['intervals_0'], fpga_events['feedback_times']),
+            'stimCenter_times': _assign_events_to_trial(
+                out['stimCenterTrigger_times'], self.frame2ttl['times'], take='first', t_trial_end=out['stimOffTrigger_times']),
+            'stimOn_times': _assign_events_to_trial(
+                out['stimOnTrigger_times'], self.frame2ttl['times'], take='first', t_trial_end=out['stimCenterTrigger_times']),
+            'stimOff_times': _assign_events_to_trial(
+                out['stimOffTrigger_times'], self.frame2ttl['times'],
+                take='first', t_trial_end=np.r_[out['intervals'][1:, 0], np.inf])
         })
         out.update({k: trials[k][ifpga] for k in trials.keys()})