Merge pull request #445 from int-brain-lab/one_globus

One globus

Author: mayofaulkner

ibllib/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.9.1"
+__version__ = "2.10.0"
 import warnings
 
 from ibllib.misc import logger_config

ibllib/atlas/atlas.py

@@ -982,4 +982,4 @@ def _download_atlas_allen(file_image, FLAT_IRON_ATLAS_REL_PATH, par):
         raise ValueError('Unrecognized file image')
 
     cache_dir = Path(par.CACHE_DIR).joinpath(FLAT_IRON_ATLAS_REL_PATH)
-    return http_download_file(url, cache_dir=cache_dir)
+    return http_download_file(url, target_dir=cache_dir)

ibllib/io/spikeglx.py

@@ -779,7 +779,7 @@ def download_raw_partial(url_cbin, url_ch, first_chunk=0, last_chunk=0, one=None
         ch_file = url_ch
     else:
         ch_file = Path(webclient.download_file(
-            url_ch, cache_dir=target_dir, clobber=True, return_md5=False))
+            url_ch, target_dir=target_dir, clobber=True, return_md5=False))
         ch_file = remove_uuid_file(ch_file)
     ch_file_stream = target_dir.joinpath(ch_file.name).with_suffix('.stream.ch')
 
@@ -836,7 +836,7 @@ def download_raw_partial(url_cbin, url_ch, first_chunk=0, last_chunk=0, one=None
     # Download the requested chunks
     cbin_local_path = webclient.download_file(
         url_cbin, chunks=(first_byte, n_bytes),
-        cache_dir=target_dir, clobber=True, return_md5=False)
+        target_dir=target_dir, clobber=True, return_md5=False)
     cbin_local_path = remove_uuid_file(cbin_local_path)
     cbin_local_path_renamed = cbin_local_path.with_suffix('.stream.cbin')
     cbin_local_path.replace(cbin_local_path_renamed)

ibllib/oneibl/aws.py

@@ -33,7 +33,7 @@ def __init__(self, s3_bucket_name=None, one=None):
     def _download_datasets(self, datasets):
 
         files = []
-        for _, d in datasets.iterrows():
+        for i, d in datasets.iterrows():
             rel_file_path = Path(d['session_path']).joinpath(d['rel_path'])
             file_path = Path(self.one.cache_dir).joinpath(rel_file_path)
             file_path.parent.mkdir(exist_ok=True, parents=True)
@@ -44,8 +44,13 @@ def _download_datasets(self, datasets):
                 _logger.info(f'{file_path} already exists wont redownload')
                 continue
 
+            if self.one._index_type() is int:
+                uuid = np2str(np.r_[i[0], i[1]])
+            elif self.one._index_type() is str:
+                uuid = i
+
             aws_path = AWS_ROOT_PATH.joinpath(
-                add_uuid_string(rel_file_path, np2str(np.r_[d.name[0], d.name[1]]))).as_posix()
+                add_uuid_string(rel_file_path, uuid)).as_posix()
             # maybe should avoid this and do a try catch instead?, see here
             # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/collections.html#filtering
             # probably better to do filter on collection ? Not for today

ibllib/oneibl/data_handlers.py

@@ -127,6 +127,7 @@ def __init__(self, session_path, signatures, one=None):
 
         # Find the lab
         labs = get_lab_from_endpoint_id(one=self.one)
+
         if len(labs) == 2:
             # for flofer lab
             subject = self.one.path2ref(self.session_path)['subject']
@@ -167,13 +168,19 @@ def setUp(self):
         target_paths = []
         source_paths = []
         self.local_paths = []
-        for _, d in df.iterrows():
+        for i, d in df.iterrows():
             sess_path = Path(rel_sess_path).joinpath(d['rel_path'])
             full_local_path = Path(self.globus.endpoints['local']['root_path']).joinpath(sess_path)
             if not full_local_path.exists():
+
+                if self.one._index_type() is int:
+                    uuid = np2str(np.r_[i[0], i[1]])
+                elif self.one._index_type() is str:
+                    uuid = i
+
                 self.local_paths.append(full_local_path)
                 target_paths.append(sess_path)
-                source_paths.append(add_uuid_string(sess_path, np2str(np.r_[d.name[0], d.name[1]])))
+                source_paths.append(add_uuid_string(sess_path, uuid))
 
         if len(target_paths) != 0:
             ts = time()
@@ -384,9 +391,15 @@ def setUp(self):
         df = super().getData()
 
         SDSC_TMP = Path(SDSC_PATCH_PATH.joinpath(self.task.__class__.__name__))
-        for _, d in df.iterrows():
+        for i, d in df.iterrows():
             file_path = Path(d['session_path']).joinpath(d['rel_path'])
-            file_uuid = add_uuid_string(file_path, np2str(np.r_[d.name[0], d.name[1]]))
+
+            if self.one._index_type() is int:
+                uuid = np2str(np.r_[i[0], i[1]])
+            elif self.one._index_type() is str:
+                uuid = i
+
+            file_uuid = add_uuid_string(file_path, uuid)
             file_link = SDSC_TMP.joinpath(file_path)
             file_link.parent.mkdir(exist_ok=True, parents=True)
             file_link.symlink_to(

ibllib/tests/test_plots.py

@@ -37,7 +37,7 @@ def _get_image(self, url):
         try:
             img_file = list(Path('/var/www/').rglob(rel_path))[0]
         except IndexError:
-            img_file = http_download_file(url, cache_dir=Path(self.tmp_dir.name), username=TEST_DB['username'],
+            img_file = http_download_file(url, target_dir=Path(self.tmp_dir.name), username=TEST_DB['username'],
                                           password=TEST_DB['password'], silent=True)
         return img_file
 

release_notes.md

@@ -1,7 +1,11 @@
-## To be added to next release
+## Release Note 2.10
+
+## Release Note 2.10.0 2022-02-11
 - Fix in EphysDLC task to not return multiple copies of outputs
 - Loading examples for different IBL data types
 - Fix for probe syncing when Nidq and probe pulses don't match
+- Account for new ONE tables in ond datahandler
+- Add bad channels plots into RawEphysQc task
 
 ## Release Note 2.9
 

requirements.txt

@@ -22,4 +22,4 @@ scipy>=1.3.0
 seaborn>=0.9.0
 tqdm>=4.32.1
 iblutil
-ONE-api
+ONE-api>=1.8.1