Deprecated ibllib.version; fix Globus patcher

Author: k1o0

ibllib/io/extractors/biased_trials.py

@@ -12,7 +12,6 @@
     StimOnTimes_deprecated, StimOnTriggerTimes, StimOnOffFreezeTimes, ItiInTimes,
     StimOffTriggerTimes, StimFreezeTriggerTimes, ErrorCueTriggerTimes, PhasePosQuiescence)
 from ibllib.io.extractors.training_wheel import Wheel
-from ibllib.misc import version
 
 
 class ContrastLR(BaseBpodTrialsExtractor):
@@ -163,7 +162,7 @@ def extract_all(session_path, save=False, bpod_trials=False, settings=False, ext
 
     base = [GoCueTriggerTimes]
     # Version check
-    if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
+    if parse_version(settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
         # We now extract a single trials table
         base.extend([
             StimOnTriggerTimes, ItiInTimes, StimOffTriggerTimes, StimFreezeTriggerTimes, ErrorCueTriggerTimes,

ibllib/io/extractors/bpod_trials.py

@@ -5,10 +5,10 @@
 import logging
 from collections import OrderedDict
 
+from pkg_resources import parse_version
 from ibllib.io.extractors import habituation_trials, training_trials, biased_trials, opto_trials
 import ibllib.io.extractors.base
 import ibllib.io.raw_data_loaders as rawio
-from ibllib.misc import version
 
 _logger = logging.getLogger('ibllib')
 
@@ -54,7 +54,8 @@ def extract_all(session_path, save=True, bpod_trials=None, settings=None):
         files_wheel = []
         wheel = OrderedDict({k: trials.pop(k) for k in tuple(trials.keys()) if 'wheel' in k})
     elif extractor_type == 'habituation':
-        if settings['IBLRIG_VERSION_TAG'] and version.le(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
+        if settings['IBLRIG_VERSION_TAG'] and \
+                parse_version(settings['IBLRIG_VERSION_TAG']) <= parse_version('5.0.0'):
             _logger.warning("No extraction of legacy habituation sessions")
             return None, None, None
         trials, files_trials = habituation_trials.extract_all(

ibllib/io/extractors/training_trials.py

@@ -6,7 +6,6 @@
 import ibllib.io.raw_data_loaders as raw
 from ibllib.io.extractors.base import BaseBpodTrialsExtractor, run_extractor_classes
 from ibllib.io.extractors.training_wheel import Wheel
-from ibllib.misc import version
 
 
 _logger = logging.getLogger('ibllib')
@@ -211,7 +210,7 @@ def get_feedback_times_ge5(session_path, data=False):
 
     def _extract(self):
         # Version check
-        if version.ge(self.settings['IBLRIG_VERSION_TAG'], '5.0.0'):
+        if parse_version(self.settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
             merge = self.get_feedback_times_ge5(self.session_path, data=self.bpod_trials)
         else:
             merge = self.get_feedback_times_lt5(self.session_path, data=self.bpod_trials)
@@ -282,7 +281,7 @@ class GoCueTriggerTimes(BaseBpodTrialsExtractor):
     var_names = 'goCueTrigger_times'
 
     def _extract(self):
-        if version.ge(self.settings['IBLRIG_VERSION_TAG'], '5.0.0'):
+        if parse_version(self.settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
             goCue = np.array([tr['behavior_data']['States timestamps']
                               ['play_tone'][0][0] for tr in self.bpod_trials])
         else:
@@ -356,7 +355,7 @@ class IncludedTrials(BaseBpodTrialsExtractor):
     var_names = 'included'
 
     def _extract(self):
-        if version.ge(self.settings['IBLRIG_VERSION_TAG'], '5.0.0'):
+        if parse_version(self.settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
             trials_included = self.get_included_trials_ge5(
                 data=self.bpod_trials, settings=self.settings)
         else:
@@ -513,7 +512,7 @@ def _extract(self):
         # Version check
         _logger.warning("Deprecation Warning: this is an old version of stimOn extraction."
                         "From version 5., use StimOnOffFreezeTimes")
-        if version.ge(self.settings['IBLRIG_VERSION_TAG'], '5.0.0'):
+        if parse_version(self.settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
             stimOn_times = self.get_stimOn_times_ge5(self.session_path, data=self.bpod_trials)
         else:
             stimOn_times = self.get_stimOn_times_lt5(self.session_path, data=self.bpod_trials)
@@ -719,7 +718,7 @@ def extract_all(session_path, save=False, bpod_trials=None, settings=None):
 
     base = [RepNum, GoCueTriggerTimes]
     # Version check
-    if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
+    if parse_version(settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
         # We now extract a single trials table
         base.extend([
             StimOnTriggerTimes, ItiInTimes, StimOffTriggerTimes, StimFreezeTriggerTimes,

ibllib/io/raw_data_loaders.py

@@ -15,12 +15,12 @@
 from pathlib import Path
 from typing import Union
 
+from pkg_resources import parse_version
 import numpy as np
 import pandas as pd
 
 from iblutil.io import jsonable
 from ibllib.io.video import assert_valid_label
-from ibllib.misc import version
 from ibllib.time import uncycle_pgts, convert_pgts
 
 _logger = logging.getLogger('ibllib')
@@ -374,7 +374,7 @@ def load_encoder_events(session_path, settings=False):
             settings = {'IBLRIG_VERSION_TAG': '0.0.0'}
     if not path:
         return None
-    if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
+    if parse_version(settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
         return _load_encoder_events_file_ge5(path)
     else:
         return _load_encoder_events_file_lt5(path)
@@ -479,7 +479,7 @@ def load_encoder_positions(session_path, settings=False):
     if not path:
         _logger.warning("No data loaded: could not find raw encoderPositions file")
         return None
-    if version.ge(settings['IBLRIG_VERSION_TAG'], '5.0.0'):
+    if parse_version(settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'):
         return _load_encoder_positions_file_ge5(path)
     else:
         return _load_encoder_positions_file_lt5(path)

ibllib/misc/version.py

@@ -1,4 +1,15 @@
 import pkg_resources
+import traceback
+import warnings
+
+for line in traceback.format_stack():
+    print(line.strip())
+
+warnings.warn(
+    'ibllib.version is deprecated and functionality will be removed! '
+    'use pkg_resources.parse_version and ibllib.__version__ instead.  See stack above.',
+    DeprecationWarning
+)
 
 
 def _compare_version_tag(v1, v2, fcn):

ibllib/oneibl/patcher.py

@@ -12,8 +12,8 @@
 from one.alf.spec import is_uuid_string
 from one import params
 from one.converters import path_from_dataset
+from one.remote import globus
 
-from ibllib.io import globus
 from ibllib.oneibl.registration import register_dataset
 
 _logger = logging.getLogger('ibllib')
@@ -97,7 +97,7 @@ def _patch_dataset(self, path, dset_id=None, dry=False, ftp=False):
             full_remote_path = PurePosixPath(FLATIRON_MOUNT, remote_path)
         if isinstance(path, WindowsPath) and not ftp:
             # On Windows replace drive map with Globus uri, e.g. C:/ -> /~/C/
-            path = '/~/' + path.as_posix().replace(':', '')
+            path = globus.as_globus_path(path)
         status = self._scp(path, full_remote_path, dry=dry)[0]
         return status
 
@@ -140,8 +140,8 @@ def patch_dataset(self, file_list, dry=False, ftp=False, **kwargs):
         Rules for creation/patching are the same that apply for registration via Alyx
         as this uses the registration endpoint to get the dataset.
         An existing file (same session and path relative to session) will be patched.
-        :param path: full file path. Must be whithin an ALF session folder (subject/date/number)
-        can also be a list of full file pathes belonging to the same session.
+        :param path: full file path. Must be within an ALF session folder (subject/date/number)
+        can also be a list of full file paths belonging to the same session.
         :param server_repository: Alyx server repository name
         :param created_by: alyx username for the dataset (optional, defaults to root)
         :param ftp: flag for case when using ftppatcher. Don't adjust windows path in
@@ -197,13 +197,12 @@ class GlobusPatcher(Patcher):
 
     """
 
-    def __init__(self, one=None, globus_client_id=None, local_endpoint=None, label='ibllib patch'):
-        assert globus_client_id
+    def __init__(self, client_name='default', one=None, label='ibllib patch'):
         assert one
-        self.local_endpoint = local_endpoint or globus.get_local_endpoint()
+        self.local_endpoint = getattr(globus.load_client_params(f'globus.{client_name}'),
+                                      'local_endpoint', globus.get_local_endpoint_id())
+        self.transfer_client = globus.create_globus_client(client_name)
         self.label = label
-        self.transfer_client = globus.login_auto(
-            globus_client_id=globus_client_id, str_app='globus/admin')
         # transfers/delete from the current computer to the flatiron: mandatory and executed first
         self.globus_transfer = globus_sdk.TransferData(
             self.transfer_client, self.local_endpoint, FLAT_IRON_GLOBUS_ID, verify_checksum=True,
@@ -296,11 +295,11 @@ def _wait_for_task(resp):
             # on an errored task
             # Out[10]: TransferResponse({'bytes_checksummed': 0, 'bytes_transferred': 0, 'canceled_by_admin': None, 'canceled_by_admin_message': None, 'command': 'API 0.10', 'completion_time': '2021-01-03T17:39:00+00:00', 'deadline': '2021-01-04T17:37:34+00:00', 'delete_destination_extra': False, 'destination_endpoint': 'simonsfoundation#ibl', 'destination_endpoint_display_name': 'IBL Flatiron SDSC Data', 'destination_endpoint_id': 'ab2d064c-413d-11eb-b188-0ee0d5d9299f', 'directories': 0, 'effective_bytes_per_second': 0, 'encrypt_data': False, 'fatal_error': {'code': 'CANCELED', 'description': 'canceled'}, 'faults': 2, 'files': 6, 'files_skipped': 0, 'files_transferred': 0, 'history_deleted': False, 'is_ok': None, 'is_paused': False, 'key': 'complete,2021-01-03T17:38:59.697413', 'label': 'test 3B analog sync patch', 'nice_status': None, 'nice_status_details': None, 'nice_status_expires_in': None, 'nice_status_short_description': None, 'owner_id': 'e633663a-8561-4a5d-ac92-f198d43b14dc', 'preserve_timestamp': False, 'recursive_symlinks': 'ignore', 'request_time': '2021-01-03T17:37:34+00:00', 'source_endpoint': 'internationalbrainlab#916c2766-bd2a-11ea-8f22-0a21f750d19b', 'source_endpoint_display_name': 'olivier_laptop', 'source_endpoint_id': '916c2766-bd2a-11ea-8f22-0a21f750d19b', 'status': 'FAILED', 'subtasks_canceled': 6, 'subtasks_expired': 0, 'subtasks_failed': 0, 'subtasks_pending': 0, 'subtasks_retrying': 0, 'subtasks_succeeded': 6, 'subtasks_total': 12, 'symlinks': 0, 'sync_level': 3, 'task_id': '5706dd2c-4dea-11eb-8ffb-0a34088e79f9', 'type': 'TRANSFER', 'username': 'internationalbrainlab', 'verify_checksum': True})  # noqa
             while True:
-                tinfo = gtc.get_task(task_id=resp['task_id'])['completion_time']
-                if tinfo['completion_time'] is not None:
+                tinfo = gtc.get_task(task_id=resp['task_id'])
+                if tinfo and tinfo['completion_time'] is not None:
                     break
                 _ = gtc.task_wait(task_id=resp['task_id'], timeout=30)
-            if tinfo['fatal_error'] is not None:
+            if tinfo and tinfo['fatal_error'] is not None:
                 raise ConnectionError(f"Globus transfer failed \n {tinfo}")
 
         # handles the transfers first

ibllib/oneibl/registration.py

@@ -4,14 +4,15 @@
 import logging
 import re
 
+from pkg_resources import parse_version
 from dateutil import parser as dateparser
 from iblutil.io import hashfile
 from one.alf.files import get_session_path
 import one.alf.exceptions as alferr
 from one.api import ONE
 
+import ibllib
 import ibllib.io.extractors.base
-from ibllib.misc import version
 import ibllib.time
 import ibllib.io.raw_data_loaders as raw
 from ibllib.io import flags
@@ -67,7 +68,7 @@ def register_dataset(file_list, one=None, created_by=None, repository=None, serv
     assert len(set([get_session_path(f) for f in file_list])) == 1
     assert all([Path(f).exists() for f in file_list])
     if versions is None:
-        versions = version.ibllib()
+        versions = ibllib.__version__
     if isinstance(versions, str):
         versions = [versions for _ in file_list]
     assert isinstance(versions, list) and len(versions) == len(file_list)
@@ -339,7 +340,7 @@ def register_session(self, ses_path, file_list=True):
               'filenames': F,
               'hashes': md5s,
               'filesizes': file_sizes,
-              'versions': [version.ibllib() for _ in F]
+              'versions': [ibllib.__version__ for _ in F]
               }
         self.one.alyx.post('/register-file', data=r_)
         return session
@@ -393,7 +394,7 @@ def _read_settings_json_compatibility_enforced(json_file):
         _logger.warning("You appear to be on an untagged version...")
         return md
     # 2018-12-05 Version 3.2.3 fixes (permanent fixes in IBL_RIG from 3.2.4 on)
-    if version.le(md['IBLRIG_VERSION_TAG'], '3.2.3'):
+    if parse_version(md['IBLRIG_VERSION_TAG']) <= parse_version('3.2.3'):
         if 'LAST_TRIAL_DATA' in md.keys():
             md.pop('LAST_TRIAL_DATA')
         if 'weighings' in md['PYBPOD_SUBJECT_EXTRA'].keys():
@@ -414,7 +415,7 @@ def _read_settings_json_compatibility_enforced(json_file):
 def rename_files_compatibility(ses_path, version_tag):
     if not version_tag:
         return
-    if version.le(version_tag, '3.2.3'):
+    if parse_version(version_tag) <= parse_version('3.2.3'):
         task_code = ses_path.glob('**/_ibl_trials.iti_duration.npy')
         for fn in task_code:
             fn.replace(fn.parent.joinpath('_ibl_trials.itiDuration.npy'))

ibllib/pipes/tasks.py

@@ -10,7 +10,7 @@
 
 from graphviz import Digraph
 
-from ibllib.misc import version
+import ibllib
 from ibllib.oneibl import data_handlers
 import one.params
 from one.api import ONE
@@ -30,7 +30,7 @@ class Task(abc.ABC):
     outputs = None  # place holder for a list of Path containing output files
     time_elapsed_secs = None
     time_out_secs = 3600 * 2  # time-out after which a task is considered dead
-    version = version.ibllib()
+    version = ibllib.__version__
     signature = {'input_files': [], 'output_files': []}  # list of tuples (filename, collection, required_flag)
     force = False  # whether or not to re-download missing input files on local server if not present
 
@@ -100,7 +100,7 @@ def run(self, **kwargs):
         _logger.info(f"Starting job {self.__class__}")
         if self.machine:
             _logger.info(f"Running on machine: {self.machine}")
-        _logger.info(f"running ibllib version {version.ibllib()}")
+        _logger.info(f"running ibllib version {ibllib.__version__}")
         # setup
         start_time = time.time()
         try:
@@ -193,7 +193,7 @@ def _run(self, overwrite=False):
         :param overwrite: (bool) if the output already exists,
         :return: out_files: files to be registered. Could be a list of files (pathlib.Path),
         a single file (pathlib.Path) an empty list [] or None.
-        Whithin the pipeline, there is a distinction between a job that returns an empty list
+        Within the pipeline, there is a distinction between a job that returns an empty list
          and a job that returns None. If the function returns None, the job will be labeled as
           "empty" status in the database, otherwise, the job has an expected behaviour of not
           returning any dataset.

ibllib/plots/snapshot.py

@@ -7,10 +7,10 @@
 
 from one.api import ONE
 from ibllib.pipes import tasks
-from ibllib.misc import version
 from one.alf.exceptions import ALFObjectNotFound
 from neuropixel import trace_header, TIP_SIZE_UM
 
+from ibllib import __version__ as ibllib_version
 from ibllib.pipes.ephys_alignment import EphysAlignment
 from ibllib.pipes.histology import interpolate_along_track
 from ibllib.atlas import AllenAtlas
@@ -36,7 +36,7 @@ def register_images(self, widths=None, function=None, extra_dict=None):
         jsons = []
         texts = []
         for f in self.outputs:
-            json_dict = dict(tag=report_tag, version=version.ibllib(),
+            json_dict = dict(tag=report_tag, version=ibllib_version,
                              function=(function or str(self.__class__).split("'")[1]), name=f.stem)
             if extra_dict is not None:
                 assert isinstance(extra_dict, dict)