Merge remote-tracking branch 'upstream/release013' into issue-829

Author: Chris Turner

CHANGELOG.md

@@ -1,6 +1,10 @@
 ## Release notes
 
 ### Current
+=======
+### 0.13.0 -- TBD
+* Support DataJoint datatype and connection plugins (#715, #729) PR 730, #735
+* Allow updating specified secondary attributes using `update1` PR #763
 * Remove python 3.5 support
 
 ### 0.12.8 -- Jan 12, 2021

datajoint.pub

@@ -0,0 +1,6 @@
+-----BEGIN PUBLIC KEY-----
+MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDUMOo2U7YQ1uOrKU/IreM3AQP2
+AXJC3au+S9W+dilxHcJ3e98bRVqrFeOofcGeRPoNc38fiLmLDUiBskJeVrpm29Wo
+AkH6yhZWk1o8NvGMhK4DLsJYlsH6tZuOx9NITKzJuOOH6X1I5Ucs7NOSKnmu7g5g
+WTT5kCgF5QAe5JN8WQIDAQAB
+-----END PUBLIC KEY-----

datajoint/attribute_adapter.py

@@ -1,5 +1,6 @@
 import re
 from .errors import DataJointError, _support_adapted_types
+from .plugin import type_plugins
 
 
 class AttributeAdapter:
@@ -38,10 +39,11 @@ def get_adapter(context, adapter_name):
         raise DataJointError('Support for Adapted Attribute types is disabled.')
     adapter_name = adapter_name.lstrip('<').rstrip('>')
     try:
-        adapter = context[adapter_name]
+        adapter = (context[adapter_name] if adapter_name in context
+                   else type_plugins[adapter_name]['object'].load())
     except KeyError:
         raise DataJointError(
-            "Attribute adapter '{adapter_name}' is not defined.".format(adapter_name=adapter_name)) from None
+            "Attribute adapter '{adapter_name}' is not defined.".format(adapter_name=adapter_name))
     if not isinstance(adapter, AttributeAdapter):
         raise DataJointError(
             "Attribute adapter '{adapter_name}' must be an instance of datajoint.AttributeAdapter".format(

datajoint/autopopulate.py

@@ -41,7 +41,7 @@ def _rename_attributes(table, props):
             parents = self.target.parents(primary=True, as_objects=True, foreign_key_info=True)
             if not parents:
                 raise DataJointError(
-                    'A relation must have primary dependencies for auto-populate to work') from None
+                    'A relation must have primary dependencies for auto-populate to work')
             self._key_source = _rename_attributes(*parents[0])
             for q in parents[1:]:
                 self._key_source *= _rename_attributes(*q)

datajoint/connection.py

@@ -11,11 +11,36 @@
 from .settings import config
 from . import errors
 from .dependencies import Dependencies
+from .plugin import connection_plugins
 
 logger = logging.getLogger(__name__)
 query_log_max_length = 300
 
 
+def get_host_hook(host_input):
+    if '://' in host_input:
+        plugin_name = host_input.split('://')[0]
+        try:
+            return connection_plugins[plugin_name]['object'].load().get_host(host_input)
+        except KeyError:
+            raise errors.DataJointError(
+                "Connection plugin '{}' not found.".format(plugin_name))
+    else:
+        return host_input
+
+
+def connect_host_hook(connection_obj):
+    if '://' in connection_obj.conn_info['host_input']:
+        plugin_name = connection_obj.conn_info['host_input'].split('://')[0]
+        try:
+            connection_plugins[plugin_name]['object'].load().connect_host(connection_obj)
+        except KeyError:
+            raise errors.DataJointError(
+                "Connection plugin '{}' not found.".format(plugin_name))
+    else:
+        connection_obj.connect()
+
+
 def translate_query_error(client_error, query):
     """
     Take client error and original query and return the corresponding DataJoint exception.
@@ -76,7 +101,8 @@ def conn(host=None, user=None, password=None, *, init_fun=None, reset=False, use
                         #encrypted-connection-options).
     """
     if not hasattr(conn, 'connection') or reset:
-        host = host if host is not None else config['database.host']
+        host_input = host if host is not None else config['database.host']
+        host = get_host_hook(host_input)
         user = user if user is not None else config['database.user']
         password = password if password is not None else config['database.password']
         if user is None:  # pragma: no cover
@@ -85,7 +111,8 @@ def conn(host=None, user=None, password=None, *, init_fun=None, reset=False, use
             password = getpass(prompt="Please enter DataJoint password: ")
         init_fun = init_fun if init_fun is not None else config['connection.init_function']
         use_tls = use_tls if use_tls is not None else config['database.use_tls']
-        conn.connection = Connection(host, user, password, None, init_fun, use_tls)
+        conn.connection = Connection(host, user, password, None, init_fun, use_tls,
+                                     host_input=host_input)
     return conn.connection
 
 
@@ -104,7 +131,8 @@ class Connection:
     :param use_tls: TLS encryption option
     """
 
-    def __init__(self, host, user, password, port=None, init_fun=None, use_tls=None):
+    def __init__(self, host, user, password, port=None, init_fun=None, use_tls=None,
+                 host_input=None):
         if ':' in host:
             # the port in the hostname overrides the port argument
             host, port = host.split(':')
@@ -115,10 +143,11 @@ def __init__(self, host, user, password, port=None, init_fun=None, use_tls=None)
         if use_tls is not False:
             self.conn_info['ssl'] = use_tls if isinstance(use_tls, dict) else {'ssl': {}}
         self.conn_info['ssl_input'] = use_tls
+        self.conn_info['host_input'] = host_input
         self.init_fun = init_fun
         print("Connecting {user}@{host}:{port}".format(**self.conn_info))
         self._conn = None
-        self.connect()
+        connect_host_hook(self)
         if self.is_connected:
             logger.info("Connected {user}@{host}:{port}".format(**self.conn_info))
             self.connection_id = self.query('SELECT connection_id()').fetchone()[0]
@@ -149,15 +178,15 @@ def connect(self):
                              "STRICT_ALL_TABLES,NO_ENGINE_SUBSTITUTION",
                     charset=config['connection.charset'],
                     **{k: v for k, v in self.conn_info.items()
-                       if k != 'ssl_input'})
+                       if k not in ['ssl_input', 'host_input']})
             except client.err.InternalError:
                 self._conn = client.connect(
                     init_command=self.init_fun,
                     sql_mode="NO_ZERO_DATE,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO,"
                              "STRICT_ALL_TABLES,NO_ENGINE_SUBSTITUTION",
                     charset=config['connection.charset'],
                     **{k: v for k, v in self.conn_info.items()
-                       if not(k == 'ssl_input' or
+                       if not(k in ['ssl_input', 'host_input'] or
                               k == 'ssl' and self.conn_info['ssl_input'] is None)})
         self._conn.autocommit(True)
 
@@ -194,7 +223,7 @@ def _execute_query(cursor, query, args, cursor_class, suppress_warnings):
                     warnings.simplefilter("ignore")
                 cursor.execute(query, args)
         except client.err.Error as err:
-            raise translate_query_error(err, query) from None
+            raise translate_query_error(err, query)
 
     def query(self, query, args=(), *, as_dict=False, suppress_warnings=True, reconnect=None):
         """
@@ -217,10 +246,10 @@ def query(self, query, args=(), *, as_dict=False, suppress_warnings=True, reconn
             if not reconnect:
                 raise
             warnings.warn("MySQL server has gone away. Reconnecting to the server.")
-            self.connect()
+            connect_host_hook(self)
             if self._in_transaction:
                 self.cancel_transaction()
-                raise errors.LostConnectionError("Connection was lost during a transaction.") from None
+                raise errors.LostConnectionError("Connection was lost during a transaction.")
             logger.debug("Re-executing")
             cursor = self._conn.cursor(cursor=cursor_class)
             self._execute_query(cursor, query, args, cursor_class, suppress_warnings)

datajoint/declare.py

@@ -45,7 +45,7 @@ def match_type(attribute_type):
     try:
         return next(category for category, pattern in TYPE_PATTERN.items() if pattern.match(attribute_type))
     except StopIteration:
-        raise DataJointError("Unsupported attribute type {type}".format(type=attribute_type)) from None
+        raise DataJointError("Unsupported attribute type {type}".format(type=attribute_type))
 
 
 logger = logging.getLogger(__name__)
@@ -135,7 +135,7 @@ def compile_foreign_key(line, context, attributes, primary_key, attr_sql, foreig
         try:
             result = foreign_key_parser_old.parseString(line)
         except pp.ParseBaseException as err:
-            raise DataJointError('Parsing error in line "%s". %s.' % (line, err)) from None
+            raise DataJointError('Parsing error in line "%s". %s.' % (line, err))
         else:
             obsolete = True
     try:
@@ -430,7 +430,7 @@ def compile_attribute(line, in_key, foreign_key_sql, context):
         match = attribute_parser.parseString(line + '#', parseAll=True)
     except pp.ParseException as err:
         raise DataJointError('Declaration error in position {pos} in line:\n  {line}\n{msg}'.format(
-            line=err.args[0], pos=err.args[1], msg=err.args[2])) from None
+            line=err.args[0], pos=err.args[1], msg=err.args[2]))
     match['comment'] = match['comment'].rstrip('#')
     if 'default' not in match:
         match['default'] = ''

datajoint/errors.py

@@ -5,11 +5,24 @@
 import os
 
 
+# --- Unverified Plugin Check ---
+class PluginWarning(Exception):
+    pass
+
+
 # --- Top Level ---
 class DataJointError(Exception):
     """
     Base class for errors specific to DataJoint internal operation.
     """
+    def __init__(self, *args):
+        from .plugin import connection_plugins, type_plugins
+        self.__cause__ = PluginWarning(
+            'Unverified DataJoint plugin detected.') if any([any(
+                [not plugins[k]['verified'] for k in plugins])
+                for plugins in [connection_plugins, type_plugins]
+                if plugins]) else None
+
     def suggest(self, *args):
         """
         regenerate the exception with additional arguments

datajoint/expression.py

@@ -129,7 +129,7 @@ def prep_value(k, v):
                     try:
                         v = uuid.UUID(v)
                     except (AttributeError, ValueError):
-                        raise DataJointError('Badly formed UUID {v} in restriction by `{k}`'.format(k=k, v=v)) from None
+                        raise DataJointError('Badly formed UUID {v} in restriction by `{k}`'.format(k=k, v=v))
                 return "X'%s'" % binascii.hexlify(v.bytes).decode()
             if isinstance(v, (datetime.date, datetime.datetime, datetime.time, decimal.Decimal)):
                 return '"%s"' % v

datajoint/external.py

@@ -230,7 +230,7 @@ def upload_filepath(self, local_filepath):
             relative_filepath = str(local_filepath.relative_to(self.spec['stage']).as_posix())
         except ValueError:
             raise DataJointError('The path {path} is not in stage {stage}'.format(
-                path=local_filepath.parent, **self.spec)) from None
+                path=local_filepath.parent, **self.spec))
         uuid = uuid_from_buffer(init_string=relative_filepath)  # hash relative path, not contents
         contents_hash = uuid_from_file(local_filepath)
 

datajoint/heading.py

@@ -250,8 +250,8 @@ def init_from_database(self, conn, database, table_name, context):
                         url = "https://docs.datajoint.io/python/admin/5-blob-config.html" \
                               "#migration-between-datajoint-v0-11-and-v0-12"
                         raise DataJointError('Legacy datatype `{type}`. Migrate your external stores to '
-                                             'datajoint 0.12: {url}'.format(url=url, **attr)) from None
-                    raise DataJointError('Unknown attribute type `{type}`'.format(**attr)) from None
+                                             'datajoint 0.12: {url}'.format(url=url, **attr))
+                    raise DataJointError('Unknown attribute type `{type}`'.format(**attr))
                 if category == 'FILEPATH' and not _support_filepath_types():
                     raise DataJointError("""
                         The filepath data type is disabled until complete validation.