Merge pull request #450 from int-brain-lab/ow

bugfix: ephys streamer always re-downloads files

Author: oliche

brainbox/io/one.py

@@ -941,7 +941,7 @@ def _get_spike_sorting_collection(self, spike_sorter='pykilosort', revision=None
         _logger.debug(f"selecting: {collection} to load amongst candidates: {self.collections}")
         return collection
 
-    def _download_spike_sorting_object(self, obj, spike_sorter='pykilosort', dataset_types=None):
+    def download_spike_sorting_object(self, obj, spike_sorter='pykilosort', dataset_types=None):
         """
         Downloads an ALF object
         :param obj: object name, str between 'spikes', 'clusters' or 'channels'
@@ -954,7 +954,8 @@ def _download_spike_sorting_object(self, obj, spike_sorter='pykilosort', dataset
         self.collection = self._get_spike_sorting_collection(spike_sorter=spike_sorter)
         _logger.debug(f"loading spike sorting from {self.collection}")
         spike_attributes, cluster_attributes = self._get_attributes(dataset_types)
-        attributes = {'spikes': spike_attributes, 'clusters': cluster_attributes, 'channels': None}
+        attributes = {'spikes': spike_attributes, 'clusters': cluster_attributes, 'channels': None,
+                      'templates': None, 'spikes_subset': None}
         self.files[obj] = self.one.load_object(self.eid, obj=obj, attribute=attributes[obj],
                                                collection=self.collection, download_only=True)
 
@@ -966,7 +967,7 @@ def download_spike_sorting(self, **kwargs):
         :return:
         """
         for obj in ['spikes', 'clusters', 'channels']:
-            self._download_spike_sorting_object(obj=obj, **kwargs)
+            self.download_spike_sorting_object(obj=obj, **kwargs)
         self.spike_sorting_path = self.files['spikes'][0].parent
 
     def load_spike_sorting(self, **kwargs):

brainbox/io/spikeglx.py

@@ -2,6 +2,7 @@
 import logging
 from pathlib import Path
 import time
+import json
 
 import numpy as np
 
@@ -126,14 +127,22 @@ def stream(pid, t0, nsecs=1, one=None, cache_folder=None, remove_cached=False, t
     if cache_folder is None:
         samples_folder = Path(one.alyx._par.CACHE_DIR).joinpath('cache', typ)
 
+    def _get_file(one_rec):
+        fpath = Path(one.alyx._par.CACHE_DIR).joinpath(
+            str(one_rec['session_path'].values[0]),
+            str(one_rec['rel_path'].values[0]))
+        if fpath.exists():
+            return fpath
+        else:
+            return one._download_datasets(one_rec)[0]
+
     eid, pname = one.pid2eid(pid)
     cbin_rec = one.list_datasets(eid, collection=f"*{pname}", filename=f'*{typ}.*bin', details=True)
     ch_rec = one.list_datasets(eid, collection=f"*{pname}", filename=f'*{typ}.ch', details=True)
     meta_rec = one.list_datasets(eid, collection=f"*{pname}", filename=f'*{typ}.meta', details=True)
-    ch_file = one._download_datasets(ch_rec)[0]
-    one._download_datasets(meta_rec)[0]
+    ch_file = _get_file(ch_rec)
+    _get_file(meta_rec)
 
-    import json
     with open(ch_file) as fid:
         chinfo = json.load(fid)
     tbounds = np.array(chinfo['chunk_bounds']) / chinfo['sample_rate']