use scratch dir for pykilosort

Author: oliche

ibllib/io/extractors/signatures.py

@@ -182,6 +182,7 @@
               ('*.ch', 'raw_ephys_data/probe*', True),
               ('*.cbin', 'raw_ephys_data/probe*', True)]
 
+
 def spike_sorting_signature(pname=None):
     pname = pname if pname is not None else "probe*"
     signature = [('*ap.meta', f'raw_ephys_data/{pname}', True),
@@ -192,5 +193,5 @@ def spike_sorting_signature(pname=None):
                  ('_spikeglx_sync.times.*', 'raw_ephys_data*', True)]
     return signature
 
-SPIKESORTING = [spike_sorting_signature()]
 
+SPIKESORTING = [spike_sorting_signature()]

ibllib/oneibl/aws.py

@@ -41,7 +41,7 @@ def _download_datasets(self, datasets):
             if file_path.exists():
                 # already downloaded, need to have some options for overwrite, clobber, look
                 # for file mismatch like in ONE
-                _logger.warning(f'{file_path} already exists wont redownload')
+                _logger.info(f'{file_path} already exists wont redownload')
                 continue
 
             aws_path = AWS_ROOT_PATH.joinpath(add_uuid_string(rel_file_path,

ibllib/pipes/ephys_preprocessing.py

@@ -134,21 +134,17 @@ def _fetch_ks2_commit_hash(repo_path):
         return info.decode("utf-8").strip()
 
     def _run_pykilosort(self, ap_file):
-        f"""
+        """
         Runs the ks2 matlab spike sorting for one probe dataset
-        the raw spike sorting output can either be with the probe (<1.5.5) or in the
-        session_path/spike_sorters/{self.SPIKE_SORTER_NAME}/probeXX folder
+        the raw spike sorting output is in session_path/spike_sorters/{self.SPIKE_SORTER_NAME}/probeXX folder
+        (discontinued support for old spike sortings in the probe folder <1.5.5)
         :return: path of the folder containing ks2 spike sorting output
         """
         self.version = self._fetch_pykilosort_version(self.PYKILOSORT_REPO)
         label = ap_file.parts[-2]  # this is usually the probe name
         sorter_dir = self.session_path.joinpath("spike_sorters", self.SPIKE_SORTER_NAME, label)
         FORCE_RERUN = False
         if not FORCE_RERUN:
-            if ap_file.parent.joinpath(f"spike_sorting_{self.SPIKE_SORTER_NAME}.log").exists():
-                _logger.info(f"Already ran: spike_sorting_{self.SPIKE_SORTER_NAME}.log"
-                             f" found for {ap_file}, skipping.")
-                return ap_file.parent
             if sorter_dir.joinpath(f"spike_sorting_{self.SPIKE_SORTER_NAME}.log").exists():
                 _logger.info(f"Already ran: spike_sorting_{self.SPIKE_SORTER_NAME}.log"
                              f" found in {sorter_dir}, skipping.")