Merge branch 'release/2.10.2'

Author: juhuntenburg

brainbox/io/one.py

@@ -941,7 +941,7 @@ def _get_spike_sorting_collection(self, spike_sorter='pykilosort', revision=None
         _logger.debug(f"selecting: {collection} to load amongst candidates: {self.collections}")
         return collection
 
-    def _download_spike_sorting_object(self, obj, spike_sorter='pykilosort', dataset_types=None):
+    def download_spike_sorting_object(self, obj, spike_sorter='pykilosort', dataset_types=None):
         """
         Downloads an ALF object
         :param obj: object name, str between 'spikes', 'clusters' or 'channels'
@@ -954,7 +954,8 @@ def _download_spike_sorting_object(self, obj, spike_sorter='pykilosort', dataset
         self.collection = self._get_spike_sorting_collection(spike_sorter=spike_sorter)
         _logger.debug(f"loading spike sorting from {self.collection}")
         spike_attributes, cluster_attributes = self._get_attributes(dataset_types)
-        attributes = {'spikes': spike_attributes, 'clusters': cluster_attributes, 'channels': None}
+        attributes = {'spikes': spike_attributes, 'clusters': cluster_attributes, 'channels': None,
+                      'templates': None, 'spikes_subset': None}
         self.files[obj] = self.one.load_object(self.eid, obj=obj, attribute=attributes[obj],
                                                collection=self.collection, download_only=True)
 
@@ -966,7 +967,7 @@ def download_spike_sorting(self, **kwargs):
         :return:
         """
         for obj in ['spikes', 'clusters', 'channels']:
-            self._download_spike_sorting_object(obj=obj, **kwargs)
+            self.download_spike_sorting_object(obj=obj, **kwargs)
         self.spike_sorting_path = self.files['spikes'][0].parent
 
     def load_spike_sorting(self, **kwargs):

brainbox/io/spikeglx.py

@@ -2,6 +2,7 @@
 import logging
 from pathlib import Path
 import time
+import json
 
 import numpy as np
 
@@ -126,14 +127,22 @@ def stream(pid, t0, nsecs=1, one=None, cache_folder=None, remove_cached=False, t
     if cache_folder is None:
         samples_folder = Path(one.alyx._par.CACHE_DIR).joinpath('cache', typ)
 
+    def _get_file(one_rec):
+        fpath = Path(one.alyx._par.CACHE_DIR).joinpath(
+            str(one_rec['session_path'].values[0]),
+            str(one_rec['rel_path'].values[0]))
+        if fpath.exists():
+            return fpath
+        else:
+            return one._download_datasets(one_rec)[0]
+
     eid, pname = one.pid2eid(pid)
     cbin_rec = one.list_datasets(eid, collection=f"*{pname}", filename=f'*{typ}.*bin', details=True)
     ch_rec = one.list_datasets(eid, collection=f"*{pname}", filename=f'*{typ}.ch', details=True)
     meta_rec = one.list_datasets(eid, collection=f"*{pname}", filename=f'*{typ}.meta', details=True)
-    ch_file = one._download_datasets(ch_rec)[0]
-    one._download_datasets(meta_rec)[0]
+    ch_file = _get_file(ch_rec)
+    _get_file(meta_rec)
 
-    import json
     with open(ch_file) as fid:
         chinfo = json.load(fid)
     tbounds = np.array(chinfo['chunk_bounds']) / chinfo['sample_rate']

ibllib/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.10.1"
+__version__ = "2.10.2"
 import warnings
 
 from ibllib.misc import logger_config

ibllib/pipes/local_server.py

@@ -16,7 +16,7 @@
 import ibllib.oneibl.registration as registration
 
 _logger = logging.getLogger('ibllib')
-LARGE_TASKS = ['EphysVideoCompress', 'TrainingVideoCompress', 'SpikeSorting']  # 'EphysDLC', 'TrainingDLC',
+LARGE_TASKS = ['EphysVideoCompress', 'TrainingVideoCompress', 'SpikeSorting', 'EphysDLC']  # 'TrainingDLC',
 
 
 def _get_pipeline_class(session_path, one):
@@ -164,6 +164,9 @@ def job_runner(subjects_path, mode='all', lab=None, dry=False, one=None, count=5
         tasks = one.alyx.rest('tasks', 'list', status='Waiting',
                               django=f'session__lab__name__in,{lab},name__in,{LARGE_TASKS}', no_cache=True)
 
+    # Order tasks by priority
+    tasks = sorted(tasks, key=lambda d: d['priority'], reverse=True)
+
     tasks_runner(subjects_path, tasks, one=one, count=count, time_out=3600, dry=dry)
 
 

release_notes.md

@@ -1,5 +1,8 @@
 ## Release Note 2.10
 
+## Release Note 2.10.2 2022-02-28
+- Small fixes to local server task queues
+
 ## Release Note 2.10.1 2022-02-22
 - Authenticate alyx user in Task class
 - Some fixes to make dlc_qc_plot in EphysPostDLC more reliable