Merge pull request #730 from guzman-raphael/plugin

dimitri-yatsenko · web-flow · commit c8710232410c · 2020-02-11T09:44:39.000-06:00
Plugin Infrastructure
diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml
@@ -37,7 +37,6 @@ services:
         pip freeze | grep datajoint;
         coveralls;
         nosetests -vsw tests --with-coverage --cover-package=datajoint;
-        # jupyter notebook;
        "
     # ports:
     #   - "8888:8888"
diff --git a/datajoint.pub b/datajoint.pub
@@ -0,0 +1,6 @@
+-----BEGIN PUBLIC KEY-----
+MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDUMOo2U7YQ1uOrKU/IreM3AQP2
+AXJC3au+S9W+dilxHcJ3e98bRVqrFeOofcGeRPoNc38fiLmLDUiBskJeVrpm29Wo
+AkH6yhZWk1o8NvGMhK4DLsJYlsH6tZuOx9NITKzJuOOH6X1I5Ucs7NOSKnmu7g5g
+WTT5kCgF5QAe5JN8WQIDAQAB
+-----END PUBLIC KEY-----
diff --git a/datajoint/attribute_adapter.py b/datajoint/attribute_adapter.py
@@ -1,5 +1,7 @@
 import re
+import os
 from .errors import DataJointError, _support_adapted_types
+from .plugin import override
 
 
 class AttributeAdapter:
@@ -30,6 +32,9 @@ def put(self, obj):
         raise NotImplementedError('Undefined attribute adapter')
 
 
+override('attribute_adapter', globals())
+
+
 def get_adapter(context, adapter_name):
     """
     Extract the AttributeAdapter object by its name from the context and validate.
@@ -38,10 +43,12 @@ def get_adapter(context, adapter_name):
         raise DataJointError('Support for Adapted Attribute types is disabled.')
     adapter_name = adapter_name.lstrip('<').rstrip('>')
     try:
-        adapter = context[adapter_name]
+        source_module_name, adapter_name = os.path.splitext(adapter_name)
+        adapter = context[source_module_name] if adapter_name == '' else getattr(
+            __import__(source_module_name, fromlist=[adapter_name[1:]]), adapter_name[1:])
     except KeyError:
         raise DataJointError(
-            "Attribute adapter '{adapter_name}' is not defined.".format(adapter_name=adapter_name)) from None
+            "Attribute adapter '{adapter_name}' is not defined.".format(adapter_name=adapter_name))
     if not isinstance(adapter, AttributeAdapter):
         raise DataJointError(
             "Attribute adapter '{adapter_name}' must be an instance of datajoint.AttributeAdapter".format(
diff --git a/datajoint/autopopulate.py b/datajoint/autopopulate.py
@@ -48,7 +48,7 @@ def parent_gen(self):
             try:
                 self._key_source = next(parents)
             except StopIteration:
-                raise DataJointError('A relation must have primary dependencies for auto-populate to work') from None
+                raise DataJointError('A table must have primary dependencies for auto-populate to work')
             for q in parents:
                 self._key_source *= q
         return self._key_source
diff --git a/datajoint/connection.py b/datajoint/connection.py
@@ -11,11 +11,23 @@
 from .settings import config
 from . import errors
 from .dependencies import Dependencies
+from .plugin import override
 
 # client errors to catch
 client_errors = (client.err.InterfaceError, client.err.DatabaseError)
 
 
+def get_host_hook(host_input):
+    return host_input
+
+
+def connect_host_hook(connection_obj):
+    connection_obj.connect()
+
+
+override('connection', globals(), ['get_host_hook', 'connect_host_hook'])
+
+
 def translate_query_error(client_error, query):
     """
     Take client error and original query and return the corresponding DataJoint exception.
@@ -76,7 +88,8 @@ def conn(host=None, user=None, password=None, *, init_fun=None, reset=False, use
                         #encrypted-connection-options).
     """
     if not hasattr(conn, 'connection') or reset:
-        host = host if host is not None else config['database.host']
+        host_input = host if host is not None else config['database.host']
+        host = get_host_hook(host_input)
         user = user if user is not None else config['database.user']
         password = password if password is not None else config['database.password']
         if user is None:  # pragma: no cover
@@ -85,7 +98,8 @@ def conn(host=None, user=None, password=None, *, init_fun=None, reset=False, use
             password = getpass(prompt="Please enter DataJoint password: ")
         init_fun = init_fun if init_fun is not None else config['connection.init_function']
         use_tls = use_tls if use_tls is not None else config['database.use_tls']
-        conn.connection = Connection(host, user, password, None, init_fun, use_tls)
+        conn.connection = Connection(host, user, password, None, init_fun, use_tls,
+                                        host_input=host_input)
     return conn.connection
 
 
@@ -104,7 +118,8 @@ class Connection:
     :param use_tls: TLS encryption option
     """
 
-    def __init__(self, host, user, password, port=None, init_fun=None, use_tls=None):
+    def __init__(self, host, user, password, port=None, init_fun=None, use_tls=None,
+                    host_input=None):
         if ':' in host:
             # the port in the hostname overrides the port argument
             host, port = host.split(':')
@@ -115,10 +130,11 @@ def __init__(self, host, user, password, port=None, init_fun=None, use_tls=None)
         if use_tls is not False:
             self.conn_info['ssl'] = use_tls if isinstance(use_tls, dict) else {'ssl': {}}
         self.conn_info['ssl_input'] = use_tls
+        self.conn_info['host_input'] = host_input
         self.init_fun = init_fun
         print("Connecting {user}@{host}:{port}".format(**self.conn_info))
         self._conn = None
-        self.connect()
+        connect_host_hook(self)
         if self.is_connected:
             logger.info("Connected {user}@{host}:{port}".format(**self.conn_info))
             self.connection_id = self.query('SELECT connection_id()').fetchone()[0]
@@ -149,15 +165,15 @@ def connect(self):
                              "STRICT_ALL_TABLES,NO_ENGINE_SUBSTITUTION",
                     charset=config['connection.charset'],
                     **{k: v for k, v in self.conn_info.items()
-                    if k != 'ssl_input'})
+                    if k not in ['ssl_input', 'host_input']})
             except client.err.InternalError:
                 self._conn = client.connect(
                     init_command=self.init_fun,
                     sql_mode="NO_ZERO_DATE,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO,"
                              "STRICT_ALL_TABLES,NO_ENGINE_SUBSTITUTION",
                     charset=config['connection.charset'],
                     **{k: v for k, v in self.conn_info.items()
-                    if not(k == 'ssl_input' or
+                    if not(k in ['ssl_input', 'host_input'] or
                     k == 'ssl' and self.conn_info['ssl_input'] is None)})
         self._conn.autocommit(True)
 
@@ -193,7 +209,7 @@ def __execute_query(cursor, query, args, cursor_class, suppress_warnings):
                     warnings.simplefilter("ignore")
                 cursor.execute(query, args)
         except client_errors as err:
-            raise translate_query_error(err, query) from None
+            raise translate_query_error(err, query)
 
     def query(self, query, args=(), *, as_dict=False, suppress_warnings=True, reconnect=None):
         """
@@ -216,10 +232,10 @@ def query(self, query, args=(), *, as_dict=False, suppress_warnings=True, reconn
             if not reconnect:
                 raise
             warnings.warn("MySQL server has gone away. Reconnecting to the server.")
-            self.connect()
+            connect_host_hook(self)
             if self._in_transaction:
                 self.cancel_transaction()
-                raise errors.LostConnectionError("Connection was lost during a transaction.") from None
+                raise errors.LostConnectionError("Connection was lost during a transaction.")
             logger.debug("Re-executing")
             cursor = self._conn.cursor(cursor=cursor_class)
             self.__execute_query(cursor, query, args, cursor_class, suppress_warnings)
diff --git a/datajoint/declare.py b/datajoint/declare.py
@@ -45,7 +45,7 @@ def match_type(attribute_type):
     try:
         return next(category for category, pattern in TYPE_PATTERN.items() if pattern.match(attribute_type))
     except StopIteration:
-        raise DataJointError("Unsupported attribute type {type}".format(type=attribute_type)) from None
+        raise DataJointError("Unsupported attribute type {type}".format(type=attribute_type))
 
 
 logger = logging.getLogger(__name__)
@@ -135,7 +135,7 @@ def compile_foreign_key(line, context, attributes, primary_key, attr_sql, foreig
         try:
             result = foreign_key_parser_old.parseString(line)
         except pp.ParseBaseException as err:
-            raise DataJointError('Parsing error in line "%s". %s.' % (line, err)) from None
+            raise DataJointError('Parsing error in line "%s". %s.' % (line, err))
         else:
             obsolete = True
     try:
@@ -430,7 +430,7 @@ def compile_attribute(line, in_key, foreign_key_sql, context):
         match = attribute_parser.parseString(line + '#', parseAll=True)
     except pp.ParseException as err:
         raise DataJointError('Declaration error in position {pos} in line:\n  {line}\n{msg}'.format(
-            line=err.args[0], pos=err.args[1], msg=err.args[2])) from None
+            line=err.args[0], pos=err.args[1], msg=err.args[2]))
     match['comment'] = match['comment'].rstrip('#')
     if 'default' not in match:
         match['default'] = ''
diff --git a/datajoint/errors.py b/datajoint/errors.py
@@ -5,11 +5,22 @@
 import os
 
 
+# --- Unverified Plugin Check ---
+class PluginWarning(Exception):
+    pass
+
+
 # --- Top Level ---
 class DataJointError(Exception):
     """
     Base class for errors specific to DataJoint internal operation.
     """
+    def __init__(self, *args):
+        from .plugin import discovered_plugins
+        self.__cause__ = PluginWarning(
+            'Unverified DataJoint plugin detected.') if discovered_plugins and any(
+            [not discovered_plugins[k]['verified'] for k in discovered_plugins]) else None
+
     def suggest(self, *args):
         """
         regenerate the exception with additional arguments
diff --git a/datajoint/expression.py b/datajoint/expression.py
@@ -129,7 +129,7 @@ def prep_value(k, v):
                     try:
                         v = uuid.UUID(v)
                     except (AttributeError, ValueError):
-                        raise DataJointError('Badly formed UUID {v} in restriction by `{k}`'.format(k=k, v=v)) from None
+                        raise DataJointError('Badly formed UUID {v} in restriction by `{k}`'.format(k=k, v=v))
                 return "X'%s'" % binascii.hexlify(v.bytes).decode()
             if isinstance(v, (datetime.date, datetime.datetime, datetime.time, decimal.Decimal)):
                 return '"%s"' % v
diff --git a/datajoint/external.py b/datajoint/external.py
@@ -227,7 +227,7 @@ def upload_filepath(self, local_filepath):
             relative_filepath = str(local_filepath.relative_to(self.spec['stage']).as_posix())
         except ValueError:
             raise DataJointError('The path {path} is not in stage {stage}'.format(
-                path=local_filepath.parent, **self.spec)) from None
+                path=local_filepath.parent, **self.spec))
         uuid = uuid_from_buffer(init_string=relative_filepath)  # hash relative path, not contents
         contents_hash = uuid_from_file(local_filepath)
 
diff --git a/datajoint/heading.py b/datajoint/heading.py
@@ -250,8 +250,8 @@ def init_from_database(self, conn, database, table_name, context):
                         url = "https://docs.datajoint.io/python/admin/5-blob-config.html" \
                               "#migration-between-datajoint-v0-11-and-v0-12"
                         raise DataJointError('Legacy datatype `{type}`. Migrate your external stores to '
-                                             'datajoint 0.12: {url}'.format(url=url, **attr)) from None
-                    raise DataJointError('Unknown attribute type `{type}`'.format(**attr)) from None
+                                             'datajoint 0.12: {url}'.format(url=url, **attr))
+                    raise DataJointError('Unknown attribute type `{type}`'.format(**attr))
                 if category == 'FILEPATH' and not _support_filepath_types():
                     raise DataJointError("""
                         The filepath data type is disabled until complete validation. 
diff --git a/datajoint/plugin.py b/datajoint/plugin.py
@@ -0,0 +1,56 @@
+import pkg_resources
+from pathlib import Path
+from cryptography.exceptions import InvalidSignature
+from setuptools_certificate import hash_pkg, verify
+from .settings import config
+
+discovered_plugins = {
+    entry_point.module_name: dict(plugon=entry_point.name, verified=False)
+    for entry_point
+    in pkg_resources.iter_entry_points('datajoint.plugins')
+    if 'plugin' not in config or entry_point.name not in config['plugin'] or
+    entry_point.module_name in config['plugin'][entry_point.name]
+}
+
+
+def _update_error_stack(plugin_name):
+    try:
+        base_name = 'datajoint'
+        base_meta = pkg_resources.get_distribution(base_name)
+        plugin_meta = pkg_resources.get_distribution(plugin_name)
+
+        data = hash_pkg(str(Path(plugin_meta.module_path, plugin_name)))
+        signature = plugin_meta.get_metadata('{}.sig'.format(plugin_name))
+        pubkey_path = str(Path(base_meta.egg_info, '{}.pub'.format(base_name)))
+        verify(pubkey_path, data, signature)
+        discovered_plugins[plugin_name]['verified'] = True
+        print('DataJoint verified plugin `{}` introduced.'.format(plugin_name))
+    except (FileNotFoundError, InvalidSignature):
+        print('Unverified plugin `{}` introduced.'.format(plugin_name))
+
+
+def override(plugin_type, context, method_list=None):
+    relevant_plugins = {
+        k: v for k, v in discovered_plugins.items() if v['plugon'] == plugin_type}
+    if relevant_plugins:
+        for module_name in relevant_plugins:
+            # import plugin
+            module = __import__(module_name)
+            module_dict = module.__dict__
+            # update error stack (if applicable)
+            _update_error_stack(module.__name__)
+            # override based on plugon preference
+            if method_list is not None:
+                new_methods = []
+                for v in method_list:
+                    try:
+                        new_methods.append(getattr(module, v))
+                    except AttributeError:
+                        pass
+                context.update(dict(zip(method_list, new_methods)))
+            else:
+                try:
+                    new_methods = module.__all__
+                except AttributeError:
+                    new_methods = [name for name in module_dict if not name.startswith('_')]
+                context.update({name: module_dict[name] for name in new_methods})
diff --git a/datajoint/s3.py b/datajoint/s3.py
@@ -32,7 +32,7 @@ def get(self, name):
         try:
             return self.client.get_object(self.bucket, str(name)).data
         except minio.error.NoSuchKey:
-            raise errors.MissingExternalFile('Missing s3 key %s' % name) from None
+            raise errors.MissingExternalFile('Missing s3 key %s' % name)
 
     def fget(self, name, local_filepath):
         """get file from object name to local filepath"""
@@ -59,7 +59,7 @@ def get_size(self, name):
         try:
             return self.client.stat_object(self.bucket, str(name)).size
         except minio.error.NoSuchKey:
-            raise errors.MissingExternalFile from None
+            raise errors.MissingExternalFile
 
     def remove_object(self, name):
         try:
diff --git a/datajoint/schema.py b/datajoint/schema.py
@@ -16,6 +16,7 @@
 from .user_tables import Part, Computed, Imported, Manual, Lookup
 from .table import lookup_class_name, Log, FreeTable
 import types
+from .plugin import override
 
 logger = logging.getLogger(__name__)
 
@@ -312,6 +313,9 @@ def create_virtual_module(module_name, schema_name, *,
     return module
 
 
+override('schema', globals())
+
+
 def list_schemas(connection=None):
     """
     :param connection: a dj.Connection object
diff --git a/datajoint/settings.py b/datajoint/settings.py
@@ -132,7 +132,7 @@ def get_store_spec(self, store):
         try:
             spec = self['stores'][store]
         except KeyError:
-            raise DataJointError('Storage {store} is requested but not configured'.format(store=store)) from None
+            raise DataJointError('Storage {store} is requested but not configured'.format(store=store))
 
         spec['subfolding'] = spec.get('subfolding', DEFAULT_SUBFOLDING)
         spec_keys = {  # REQUIRED in uppercase and allowed in lowercase
@@ -143,7 +143,7 @@ def get_store_spec(self, store):
             spec_keys = spec_keys[spec.get('protocol', '').lower()]
         except KeyError:
             raise DataJointError(
-                'Missing or invalid protocol in dj.config["stores"]["{store}"]'.format(store=store)) from None
+                'Missing or invalid protocol in dj.config["stores"]["{store}"]'.format(store=store))
 
         # check that all required keys are present in spec
         try:
diff --git a/datajoint/table.py b/datajoint/table.py
@@ -260,7 +260,7 @@ def make_placeholder(name, value):
                                 value = uuid.UUID(value)
                             except (AttributeError, ValueError):
                                 raise DataJointError(
-                                    'badly formed UUID value {v} for attribute `{n}`'.format(v=value, n=name)) from None
+                                    'badly formed UUID value {v} for attribute `{n}`'.format(v=value, n=name))
                         value = value.bytes
                     elif attr.is_blob:
                         value = blob.pack(value)
@@ -341,9 +341,9 @@ def check_fields(fields):
                 self.connection.query(query, args=list(
                     itertools.chain.from_iterable((v for v in r['values'] if v is not None) for r in rows)))
             except UnknownAttributeError as err:
-                raise err.suggest('To ignore extra fields in insert, set ignore_extra_fields=True') from None
+                raise err.suggest('To ignore extra fields in insert, set ignore_extra_fields=True')
             except DuplicateError as err:
-                raise err.suggest('To ignore duplicate entries in insert, set skip_duplicates=True') from None
+                raise err.suggest('To ignore duplicate entries in insert, set skip_duplicates=True')
 
     def delete_quick(self, get_count=False):
         """
diff --git a/datajoint/version.py b/datajoint/version.py
@@ -1,3 +1,3 @@
-__version__ = "0.12.4"
+__version__ = "0.12.5"
 
 assert len(__version__) <= 10  # The log table limits version to the 10 characters
diff --git a/requirements.txt b/requirements.txt
@@ -8,3 +8,5 @@ networkx
 pydot
 minio
 matplotlib
+cryptography
+setuptools_certificate
diff --git a/setup.py b/setup.py
@@ -31,5 +31,7 @@
     keywords='database organization',
     packages=find_packages(exclude=['contrib', 'docs', 'tests*']),
     install_requires=requirements,
-    python_requires='~={}.{}'.format(*min_py_version)
+    python_requires='~={}.{}'.format(*min_py_version),
+    setup_requires=['setuptools_certificate'],
+    pubkey_path='./datajoint.pub'
 )
diff --git a/tests/__init__.py b/tests/__init__.py
diff --git a/tests/test_adapted_attributes.py b/tests/test_adapted_attributes.py
diff --git a/tests/test_plugin.py b/tests/test_plugin.py
diff --git a/tests/test_privileges.py b/tests/test_privileges.py

Original file line number	Diff line number	Diff line change
`@@ -37,7 +37,6 @@ services:`
`37`	`37`	`pip freeze \| grep datajoint;`
`38`	`38`	`coveralls;`
`39`	`39`	`nosetests -vsw tests --with-coverage --cover-package=datajoint;`
`40`		`- # jupyter notebook;`
`41`	`40`	`"`
`42`	`41`	`# ports:`
`43`	`42`	`# - "8888:8888"`