Merge branch 'release/2.31.0'

mayofaulkner · mayofaulkner · commit 69121987bd3d · 2024-02-26T16:54:57.000Z
diff --git a/brainbox/behavior/dlc.py b/brainbox/behavior/dlc.py
@@ -9,7 +9,7 @@
 import scipy.interpolate as interpolate
 from scipy.stats import zscore
 
-from neurodsp.smooth import smooth_interpolate_savgol
+from ibldsp.smooth import smooth_interpolate_savgol
 from iblutil.numerical import bincount2D
 import brainbox.behavior.wheel as bbox_wheel
 
diff --git a/brainbox/metrics/electrode_drift.py b/brainbox/metrics/electrode_drift.py
@@ -1,6 +1,6 @@
 import numpy as np
 
-from neurodsp import smooth, utils, fourier
+from ibldsp import smooth, utils, fourier
 from iblutil.numerical import bincount2D
 
 
diff --git a/brainbox/metrics/single_units.py b/brainbox/metrics/single_units.py
@@ -981,7 +981,7 @@ def quick_unit_metrics(spike_clusters, spike_times, spike_amps, spike_depths,
     r.amp_std_dB[ir] = np.array(camp['log_amps'].std())
     srp = metrics.slidingRP_all(spikeTimes=spike_times, spikeClusters=spike_clusters,
                                 **{'sampleRate': 30000, 'binSizeCorr': 1 / 30000})
-    r.slidingRP_viol[srp['cidx']] = srp['value']
+    r.slidingRP_viol[ir] = srp['value']
 
     # loop over each cluster to compute the rest of the metrics
     for ic in np.arange(nclust):
diff --git a/examples/archive/ibllib/synchronisation_ephys.py b/examples/archive/ibllib/synchronisation_ephys.py
@@ -1,4 +1,4 @@
-import neurodsp.utils
+import ibldsp.utils
 import spikeglx
 import ibllib.io.extractors.ephys_fpga
 
@@ -15,7 +15,7 @@
 
 # if the data is needed as well, loop over the file
 # raw data contains raw ephys traces, while raw_sync contains the 16 sync traces
-wg = neurodsp.utils.WindowGenerator(sr.ns, BATCH_SIZE_SAMPLES, overlap=1)
+wg = ibldsp.utils.WindowGenerator(sr.ns, BATCH_SIZE_SAMPLES, overlap=1)
 for first, last in wg.firstlast:
     rawdata, rawsync = sr.read_samples(first, last)
     wg.print_progress()
diff --git a/examples/loading_data/loading_raw_ephys_data.ipynb b/examples/loading_data/loading_raw_ephys_data.ipynb
@@ -236,7 +236,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from neurodsp.voltage import destripe\n",
+    "from ibldsp.voltage import destripe\n",
     "# Reminder : If not done before, remove first the sync channel from raw data\n",
     "# Apply destriping algorithm to data\n",
     "destriped = destripe(raw_ap, fs=sr_ap.fs)"
@@ -445,7 +445,7 @@
    "source": [
     "## Useful modules\n",
     "* [ibllib.io.spikeglx](https://int-brain-lab.github.io/ibl-neuropixel/_autosummary/spikeglx.html)\n",
-    "* [ibllib.voltage.dsp](https://int-brain-lab.github.io/ibl-neuropixel/_autosummary/neurodsp.voltage.html)\n",
+    "* [ibllib.voltage.dsp](https://int-brain-lab.github.io/ibl-neuropixel/_autosummary/ibldsp.voltage.html)\n",
     "* [brainbox.io.spikeglx.stream](https://int-brain-lab.github.io/iblenv/_autosummary/brainbox.io.spikeglx.html#brainbox.io.spikeglx.stream)\n",
     "* [viewephys](https://github.com/oliche/viewephys) to visualise raw data snippets (Note: this package is not within `ibllib` but standalone)"
    ]
diff --git a/examples/one/histology/coverage_map.py b/examples/one/histology/coverage_map.py
@@ -1,7 +1,7 @@
 import matplotlib.pyplot as plt
 import numpy as np
 from one.api import ONE
-from neurodsp.utils import fcn_cosine
+from ibldsp.utils import fcn_cosine
 
 import iblatlas.atlas as atlas
 from ibllib.pipes.histology import coverage
diff --git a/ibllib/__init__.py b/ibllib/__init__.py
@@ -2,7 +2,7 @@
 import logging
 import warnings
 
-__version__ = '2.30.0'
+__version__ = '2.31.0'
 warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib')
 
 # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev
diff --git a/ibllib/ephys/ephysqc.py b/ibllib/ephys/ephysqc.py
@@ -11,7 +11,7 @@
 from iblutil.util import Bunch
 import spikeglx
 import neuropixel
-from neurodsp import fourier, utils, voltage
+from ibldsp import fourier, utils, voltage
 from tqdm import tqdm
 
 from brainbox.io.spikeglx import Streamer
diff --git a/ibllib/ephys/spikes.py b/ibllib/ephys/spikes.py
@@ -142,7 +142,7 @@ def ks2_to_alf(ks_path, bin_path, out_path, bin_file=None, ampfactor=1, label=No
     """
     m = ephysqc.phy_model_from_ks2_path(ks2_path=ks_path, bin_path=bin_path, bin_file=bin_file)
     ac = phylib.io.alf.EphysAlfCreator(m)
-    ac.convert(out_path, label=label, force=force, ampfactor=ampfactor)
+    ac.convert(out_path, label=label, force=force, ampfactor=float(ampfactor))
 
 
 def ks2_to_tar(ks_path, out_path, force=False):
diff --git a/ibllib/ephys/sync_probes.py b/ibllib/ephys/sync_probes.py
@@ -199,7 +199,7 @@ def sync_probe_front_times(t, tref, sr, display=False, type='smooth', tol=2.0):
         to the sampling rate of digital channels. The residual is fit using frequency domain
         smoothing
         """
-        import neurodsp.fourier
+        import ibldsp.fourier
         CAMERA_UPSAMPLING_RATE_HZ = 300
         PAD_LENGTH_SECS = 60
         STAT_LENGTH_SECS = 30  # median length to compute padding value
@@ -214,7 +214,7 @@ def sync_probe_front_times(t, tref, sr, display=False, type='smooth', tol=2.0):
         res_filt = np.pad(res_upsamp, lpad, mode='median',
                           stat_length=CAMERA_UPSAMPLING_RATE_HZ * STAT_LENGTH_SECS)
         fbounds = [0.001, 0.002]
-        res_filt = neurodsp.fourier.lp(res_filt, 1 / CAMERA_UPSAMPLING_RATE_HZ, fbounds)[lpad[0]:-lpad[1]]
+        res_filt = ibldsp.fourier.lp(res_filt, 1 / CAMERA_UPSAMPLING_RATE_HZ, fbounds)[lpad[0]:-lpad[1]]
         tout = np.arange(0, np.max(tref) + SYNC_SAMPLING_RATE_SECS, 20)
         sync_points = np.c_[tout, np.polyval(pol, tout) + np.interp(tout, t_upsamp, res_filt)]
         if display:
diff --git a/ibllib/io/extractors/camera.py b/ibllib/io/extractors/camera.py
@@ -10,7 +10,7 @@
 import matplotlib.pyplot as plt
 from iblutil.util import range_str
 
-import neurodsp.utils as dsp
+import ibldsp.utils as dsp
 from ibllib.plots import squares, vertical_lines
 from ibllib.io.video import assert_valid_label, VideoStreamer
 from iblutil.numerical import within_ranges
diff --git a/ibllib/io/extractors/ephys_fpga.py b/ibllib/io/extractors/ephys_fpga.py
@@ -44,7 +44,7 @@
 from packaging import version
 
 import spikeglx
-import neurodsp.utils
+import ibldsp.utils
 import one.alf.io as alfio
 from iblutil.util import Bunch
 from iblutil.spacer import Spacer
@@ -160,11 +160,11 @@ def _sync_to_alf(raw_ephys_apfile, output_path=None, save=False, parts=''):
     file_ftcp = Path(output_path).joinpath(f'fronts_times_channel_polarity{uuid.uuid4()}.bin')
 
     # loop over chunks of the raw ephys file
-    wg = neurodsp.utils.WindowGenerator(sr.ns, int(SYNC_BATCH_SIZE_SECS * sr.fs), overlap=1)
+    wg = ibldsp.utils.WindowGenerator(sr.ns, int(SYNC_BATCH_SIZE_SECS * sr.fs), overlap=1)
     fid_ftcp = open(file_ftcp, 'wb')
     for sl in wg.slice:
         ss = sr.read_sync(sl)
-        ind, fronts = neurodsp.utils.fronts(ss, axis=0)
+        ind, fronts = ibldsp.utils.fronts(ss, axis=0)
         # a = sr.read_sync_analog(sl)
         sav = np.c_[(ind[0, :] + sl.start) / sr.fs, ind[1, :], fronts.astype(np.double)]
         sav.tofile(fid_ftcp)
@@ -775,7 +775,7 @@ def _extract(self, sync=None, chmap=None, sync_collection='raw_ephys_data',
             bpod_start = self.bpod_trials['intervals'][:, 0]
             if len(t_trial_start) > len(bpod_start) / 2:  # if least half the trial start TTLs detected
                 _logger.warning('Attempting to get protocol period from aligning trial start TTLs')
-                fcn, *_ = neurodsp.utils.sync_timestamps(bpod_start, t_trial_start)
+                fcn, *_ = ibldsp.utils.sync_timestamps(bpod_start, t_trial_start)
                 buffer = 2.5  # the number of seconds to include before/after task
                 start, end = fcn(self.bpod_trials['intervals'].flat[[0, -1]])
                 tmin = min(sync['times'][0], start - buffer)
@@ -1202,7 +1202,7 @@ def sync_bpod_clock(bpod_trials, fpga_trials, sync_field):
                 bpod_fpga_timestamps[i] = trials[sync_field]
 
         # Sync the two timestamps
-        fcn, drift, ibpod, ifpga = neurodsp.utils.sync_timestamps(*bpod_fpga_timestamps, return_indices=True)
+        fcn, drift, ibpod, ifpga = ibldsp.utils.sync_timestamps(*bpod_fpga_timestamps, return_indices=True)
 
         # If it's drifting too much throw warning or error
         _logger.info('N trials: %i bpod, %i FPGA, %i merged, sync %.5f ppm',
diff --git a/ibllib/io/extractors/fibrephotometry.py b/ibllib/io/extractors/fibrephotometry.py
@@ -25,7 +25,7 @@
 from ibllib.io.extractors.base import BaseExtractor
 from ibllib.io.raw_daq_loaders import load_channels_tdms, load_raw_daq_tdms
 from ibllib.io.extractors.training_trials import GoCueTriggerTimes
-from neurodsp.utils import rises, sync_timestamps
+from ibldsp.utils import rises, sync_timestamps
 
 _logger = logging.getLogger(__name__)
 
diff --git a/ibllib/io/extractors/habituation_trials.py b/ibllib/io/extractors/habituation_trials.py
@@ -2,6 +2,8 @@
 import logging
 import numpy as np
 
+from packaging import version
+
 import ibllib.io.raw_data_loaders as raw
 from ibllib.io.extractors.base import BaseBpodTrialsExtractor, run_extractor_classes
 from ibllib.io.extractors.biased_trials import ContrastLR
@@ -26,10 +28,13 @@ def _extract(self) -> dict:
         """
         Extract the Bpod trial events.
 
-        The Bpod state machine for this task has extremely misleading names! The 'iti' state is
-        actually the delay between valve open and trial end (the stimulus is still present during
-        this period), and the 'trial_start' state is actually the ITI during which there is a 1s
-        Bpod TTL and gray screen period.
+        For iblrig versions < 8.13 the Bpod state machine for this task had extremely misleading names!
+        The 'iti' state was actually the delay between valve close and trial end (the stimulus is
+        still present during this period), and the 'trial_start' state is actually the ITI during
+        which there is a 1s Bpod TTL and gray screen period.
+
+        In version 8.13 and later, the 'iti' state was renamed to 'post_reward' and 'trial_start'
+        was renamed to 'iti'.
 
         Returns
         -------
@@ -57,12 +62,17 @@ def _extract(self) -> dict:
                                        bpod_trials=self.bpod_trials, settings=self.settings, task_collection=self.task_collection)
 
         """
-        The 'trial_start' state is in fact the 1s grey screen period, therefore the first timestamp
-        is really the end of the previous trial and also the stimOff trigger time. The second
-        timestamp is the true trial start time.
+        The 'trial_start'/'iti' state is in fact the 1s grey screen period, therefore the first
+        timestamp is really the end of the previous trial and also the stimOff trigger time. The
+        second timestamp is the true trial start time. This state was renamed in version 8.13.
         """
+        state_names = self.bpod_trials[0]['behavior_data']['States timestamps'].keys()
+        rig_version = version.parse(self.settings['IBLRIG_VERSION'])
+        legacy_state_machine = 'post_reward' not in state_names and 'trial_start' in state_names
+
+        key = 'iti' if (rig_version >= version.parse('8.13') and not legacy_state_machine) else 'trial_start'
         (_, *ends), starts = zip(*[
-            t['behavior_data']['States timestamps']['trial_start'][-1] for t in self.bpod_trials]
+            t['behavior_data']['States timestamps'][key][-1] for t in self.bpod_trials]
         )
 
         # StimOffTrigger times
diff --git a/ibllib/io/extractors/mesoscope.py b/ibllib/io/extractors/mesoscope.py
@@ -105,7 +105,7 @@ class TimelineTrials(FpgaTrials):
     timeline = None
     """one.alf.io.AlfBunch: The timeline data object."""
 
-    sync_field = 'itiIn_times'  # trial start events
+    sync_field = 'itiIn_times'
     """str: The trial event to synchronize (must be present in extracted trials)."""
 
     def __init__(self, *args, sync_collection='raw_sync_data', **kwargs):
diff --git a/ibllib/io/extractors/training_audio.py b/ibllib/io/extractors/training_audio.py
@@ -9,8 +9,8 @@
 from scipy.io import wavfile
 
 
-from neurodsp.utils import WindowGenerator
-from neurodsp import fourier
+from ibldsp.utils import WindowGenerator
+from ibldsp import fourier
 import ibllib.io.raw_data_loaders as ioraw
 from ibllib.io.extractors.training_trials import GoCueTimes
 
diff --git a/ibllib/io/extractors/training_wheel.py b/ibllib/io/extractors/training_wheel.py
@@ -4,7 +4,7 @@
 import numpy as np
 from scipy import interpolate
 
-from neurodsp.utils import sync_timestamps
+from ibldsp.utils import sync_timestamps
 from ibllib.io.extractors.base import BaseBpodTrialsExtractor, run_extractor_classes
 import ibllib.io.raw_data_loaders as raw
 from ibllib.misc import structarr
diff --git a/ibllib/io/extractors/video_motion.py b/ibllib/io/extractors/video_motion.py
@@ -15,7 +15,7 @@
 from pathlib import Path
 from joblib import Parallel, delayed, cpu_count
 
-from neurodsp.utils import WindowGenerator
+from ibldsp.utils import WindowGenerator
 from one.api import ONE
 import ibllib.io.video as vidio
 from iblutil.util import Bunch
diff --git a/ibllib/io/extractors/widefield.py b/ibllib/io/extractors/widefield.py
@@ -11,8 +11,17 @@
 from ibllib.io.extractors.ephys_fpga import get_sync_fronts, get_sync_and_chn_map
 from ibllib.io.video import get_video_meta
 
-import wfield.cli as wfield_cli
-from labcams.io import parse_cam_log
+_logger = logging.getLogger('ibllib')
+
+try:
+    import wfield.cli as wfield_cli
+except ImportError:
+    _logger.warning('wfield not installed')
+
+try:
+    from labcams.io import parse_cam_log
+except ImportError:
+    _logger.warning('labcams not installed')
 
 _logger = logging.getLogger('ibllib')
 
diff --git a/ibllib/io/raw_daq_loaders.py b/ibllib/io/raw_daq_loaders.py
@@ -6,7 +6,7 @@
 
 import nptdms
 import numpy as np
-import neurodsp.utils
+import ibldsp.utils
 import one.alf.io as alfio
 import one.alf.exceptions as alferr
 from one.alf.spec import to_alf
@@ -134,7 +134,7 @@ def load_sync_tdms(path, sync_map, fs=None, threshold=2.5, floor_percentile=10):
     logger.info(f'estimated analogue channel DC Offset approx. {np.mean(offset):.2f}')
     analogue -= offset
     ttl = analogue > threshold
-    ind, sign = neurodsp.utils.fronts(ttl.astype(int))
+    ind, sign = ibldsp.utils.fronts(ttl.astype(int))
     try:  # attempt to get the times from the meta data
         times = np.vstack([ch.time_track() for ch in raw_channels])
         times = times[tuple(ind)]
@@ -276,8 +276,8 @@ def extract_sync_timeline(timeline, chmap=None, floor_percentile=10, threshold=N
             step = threshold.get(label) if isinstance(threshold, dict) else threshold
             if step is None:
                 step = np.max(raw - offset) / 2
-            iup = neurodsp.utils.rises(raw - offset, step=step, analog=True)
-            idown = neurodsp.utils.falls(raw - offset, step=step, analog=True)
+            iup = ibldsp.utils.rises(raw - offset, step=step, analog=True)
+            idown = ibldsp.utils.falls(raw - offset, step=step, analog=True)
             pol = np.r_[np.ones_like(iup), -np.ones_like(idown)].astype('i1')
             ind = np.r_[iup, idown]
 
diff --git a/ibllib/pipes/dynamic_pipeline.py b/ibllib/pipes/dynamic_pipeline.py
@@ -2,6 +2,23 @@
 
 The principal function here is `make_pipeline` which reads an `_ibl_experiment.description.yaml`
 file and determines the set of tasks required to preprocess the session.
+
+In the experiment description file there is a 'tasks' key that defines each task protocol and the
+location of the raw data (i.e. task collection). The protocol subkey may contain an 'extractors'
+field that should contain a list of dynamic pipeline task class names for extracting the task data.
+These must be subclasses of the :class:`ibllib.pipes.base_tasks.DynamicTask` class. If the
+extractors key is absent or empty, the tasks are chosen based on the sync label and protocol name.
+
+NB: The standard behvaiour extraction task classes (e.g.
+:class:`ibllib.pipes.behaviour_tasks.ChoiceWorldTrialsBpod` and :class:`ibllib.pipes.behaviour_tasks.ChoiceWorldTrialsNidq`)
+handle the clock synchronization, behaviour plots and QC. This is typically independent of the Bpod
+trials extraction (i.e. extraction of trials data from the Bpod raw data, in Bpod time). The Bpod
+trials extractor class is determined by the :func:`ibllib.io.extractors.base.protocol2extractor`
+map. IBL protocols may be added to the ibllib.io.extractors.task_extractor_map.json file, while
+non-IBL ones should be in projects.base.task_extractor_map.json file located in the personal
+projects repo. The Bpod trials extractor class must be a subclass of the
+:class:`ibllib.io.extractors.base.BaseBpodTrialsExtractor` class, and located in either the
+personal projects repo or in :py:mod:`ibllib.io.extractors.bpod_trials` module.
 """
 import logging
 import re
diff --git a/ibllib/pipes/ephys_preprocessing.py b/ibllib/pipes/ephys_preprocessing.py
@@ -18,7 +18,7 @@
 import packaging.version
 
 import one.alf.io as alfio
-from neurodsp.utils import rms
+from ibldsp.utils import rms
 import spikeglx
 
 from ibllib.misc import check_nvidia_driver
diff --git a/ibllib/pipes/ephys_tasks.py b/ibllib/pipes/ephys_tasks.py
@@ -10,7 +10,7 @@
 import pandas as pd
 import spikeglx
 import neuropixel
-from neurodsp.utils import rms
+from ibldsp.utils import rms
 import one.alf.io as alfio
 
 from ibllib.misc import check_nvidia_driver
diff --git a/ibllib/pipes/histology.py b/ibllib/pipes/histology.py
@@ -11,7 +11,7 @@
 from ibllib.ephys.spikes import probes_description as extract_probes
 from ibllib.qc import base
 
-from neurodsp.utils import fcn_cosine
+from ibldsp.utils import fcn_cosine
 
 
 _logger = logging.getLogger(__name__)
diff --git a/ibllib/pipes/training_status.py b/ibllib/pipes/training_status.py
diff --git a/ibllib/pipes/widefield_tasks.py b/ibllib/pipes/widefield_tasks.py
diff --git a/ibllib/plots/figures.py b/ibllib/plots/figures.py
diff --git a/ibllib/plots/misc.py b/ibllib/plots/misc.py
diff --git a/ibllib/qc/camera.py b/ibllib/qc/camera.py
diff --git a/ibllib/qc/task_metrics.py b/ibllib/qc/task_metrics.py
diff --git a/ibllib/qc/task_qc_viewer/task_qc.py b/ibllib/qc/task_qc_viewer/task_qc.py
diff --git a/ibllib/tests/qc/test_task_metrics.py b/ibllib/tests/qc/test_task_metrics.py
diff --git a/ibllib/tests/test_ephys.py b/ibllib/tests/test_ephys.py
diff --git a/release_notes.md b/release_notes.md
diff --git a/requirements.txt b/requirements.txt
diff --git a/setup.py b/setup.py
