Merge pull request #735 from guzman-raphael/plugin2

Restructure Plugin Infrastructure

Author: dimitri-yatsenko

CHANGELOG.md

@@ -1,5 +1,15 @@
 ## Release notes
 
+### 0.12.5plug -- Feb 24, 2020
+* Support DataJoint datatype and connection plugins (#715, #729) PR 730, #735
+
+### 0.12.5 -- Feb 24, 2020
+* Rename module `dj.schema` into `dj.schemas`. `dj.schema` remains an alias for class `dj.Schema`. (#731) PR #732
+* `dj.create_virtual_module` is now called `dj.VirtualModule` (#731) PR #732
+* Bugfix - SSL `KeyError` on failed connection (#716) PR #725
+* Bugfix - Unable to run unit tests using nosetests (#723) PR #724
+* Bugfix - `suppress_errors` does not suppress loss of connection error (#720) PR #721
+
 ### 0.12.4 -- Jan 14, 2020
 * Support for simple scalar datatypes in blobs (#690) PR #709
 * Add support for the `serial` data type in declarations: alias for `bigint unsigned auto_increment` PR #713

LNX-docker-compose.yml

@@ -14,10 +14,10 @@ services:
       fakeservices.datajoint.io:
         condition: service_healthy
     environment:
-      - DJ_HOST=db
+      - DJ_HOST=fakeservices.datajoint.io
       - DJ_USER=root
       - DJ_PASS=simple
-      - DJ_TEST_HOST=db
+      - DJ_TEST_HOST=fakeservices.datajoint.io
       - DJ_TEST_USER=datajoint
       - DJ_TEST_PASSWORD=datajoint
       - S3_ENDPOINT=fakeservices.datajoint.io:9000
@@ -56,10 +56,10 @@ services:
     #   - ./mysql/data:/var/lib/mysql
   minio:
     <<: *net
+    image: minio/minio:$MINIO_VER
     environment:
       - MINIO_ACCESS_KEY=datajoint
       - MINIO_SECRET_KEY=datajoint
-    image: minio/minio:$MINIO_VER
     # ports:
     #   - "9000:9000"
     # volumes:
@@ -78,6 +78,7 @@ services:
       - URL=datajoint.io
       - SUBDOMAINS=fakeservices
       - MINIO_SERVER=http://minio:9000
+      - MYSQL_SERVER=db:3306
     entrypoint: /entrypoint.sh
     healthcheck:
       test: wget --quiet --tries=1 --spider https://fakeservices.datajoint.io:443/minio/health/live || exit 1
@@ -87,8 +88,10 @@ services:
     # ports:
     #   - "9000:9000"
     #   - "443:443"
+    #   - "3306:3306"
     volumes:
       - ./tests/nginx/base.conf:/base.conf
+      - ./tests/nginx/nginx.conf:/nginx.conf
       - ./tests/nginx/entrypoint.sh:/entrypoint.sh
       - ./tests/nginx/fullchain.pem:/certs/fullchain.pem
       - ./tests/nginx/privkey.pem:/certs/privkey.pem

datajoint/__init__.py

@@ -18,8 +18,8 @@
 __date__ = "February 7, 2019"
 __all__ = ['__author__', '__version__',
            'config', 'conn', 'Connection',
-           'schema', 'create_virtual_module', 'list_schemas',
-           'Table', 'FreeTable',
+           'Schema', 'schema', 'VirtualModule', 'create_virtual_module',
+           'list_schemas', 'Table', 'FreeTable',
            'Manual', 'Lookup', 'Imported', 'Computed', 'Part',
            'Not', 'AndList', 'U', 'Diagram', 'Di', 'ERD',
            'set_password', 'kill',
@@ -29,8 +29,8 @@
 from .version import __version__
 from .settings import config
 from .connection import conn, Connection
-from .schema import Schema as schema
-from .schema import create_virtual_module, list_schemas
+from .schemas import Schema
+from .schemas import VirtualModule, list_schemas
 from .table import Table, FreeTable
 from .user_tables import Manual, Lookup, Imported, Computed, Part
 from .expression import Not, AndList, U
@@ -43,4 +43,6 @@
 from .errors import DataJointError
 from .migrate import migrate_dj011_external_blob_storage_to_dj012
 
-ERD = Di = Diagram   # Aliases for Diagram
+ERD = Di = Diagram                      # Aliases for Diagram
+schema = Schema                         # Aliases for Schema
+create_virtual_module = VirtualModule   # Aliases for VirtualModule

datajoint/attribute_adapter.py

@@ -1,7 +1,6 @@
 import re
-import os
 from .errors import DataJointError, _support_adapted_types
-from .plugin import override
+from .plugin import type_plugins
 
 
 class AttributeAdapter:
@@ -32,9 +31,6 @@ def put(self, obj):
         raise NotImplementedError('Undefined attribute adapter')
 
 
-override('attribute_adapter', globals())
-
-
 def get_adapter(context, adapter_name):
     """
     Extract the AttributeAdapter object by its name from the context and validate.
@@ -43,9 +39,8 @@ def get_adapter(context, adapter_name):
         raise DataJointError('Support for Adapted Attribute types is disabled.')
     adapter_name = adapter_name.lstrip('<').rstrip('>')
     try:
-        source_module_name, adapter_name = os.path.splitext(adapter_name)
-        adapter = context[source_module_name] if adapter_name == '' else getattr(
-            __import__(source_module_name, fromlist=[adapter_name[1:]]), adapter_name[1:])
+        adapter = (context[adapter_name] if adapter_name in context
+                    else type_plugins[adapter_name]['object'].load())
     except KeyError:
         raise DataJointError(
             "Attribute adapter '{adapter_name}' is not defined.".format(adapter_name=adapter_name))

datajoint/connection.py

@@ -11,21 +11,34 @@
 from .settings import config
 from . import errors
 from .dependencies import Dependencies
-from .plugin import override
+from .plugin import connection_plugins
 
 # client errors to catch
 client_errors = (client.err.InterfaceError, client.err.DatabaseError)
 
 
 def get_host_hook(host_input):
-    return host_input
+    if '://' in host_input:
+        plugin_name = host_input.split('://')[0]
+        try:
+            return connection_plugins[plugin_name]['object'].load().get_host(host_input)
+        except KeyError:
+            raise DataJointError(
+                "Connection plugin '{}' not found.".format(plugin_name))
+    else:
+        return host_input
 
 
 def connect_host_hook(connection_obj):
-    connection_obj.connect()
-
-
-override('connection', globals(), ['get_host_hook', 'connect_host_hook'])
+    if '://' in connection_obj.conn_info['host_input']:
+        plugin_name = connection_obj.conn_info['host_input'].split('://')[0]
+        try:
+            connection_plugins[plugin_name]['object'].load().connect_host(connection_obj)
+        except KeyError:
+            raise DataJointError(
+                "Connection plugin '{}' not found.".format(plugin_name))
+    else:
+        connection_obj.connect()
 
 
 def translate_query_error(client_error, query):

datajoint/errors.py

@@ -16,10 +16,12 @@ class DataJointError(Exception):
     Base class for errors specific to DataJoint internal operation.
     """
     def __init__(self, *args):
-        from .plugin import discovered_plugins
+        from .plugin import connection_plugins, type_plugins
         self.__cause__ = PluginWarning(
-            'Unverified DataJoint plugin detected.') if discovered_plugins and any(
-            [not discovered_plugins[k]['verified'] for k in discovered_plugins]) else None
+            'Unverified DataJoint plugin detected.') if any([any(
+            [not plugins[k]['verified'] for k in plugins])
+            for plugins in [connection_plugins, type_plugins]
+            if plugins]) else None
 
     def suggest(self, *args):
         """

datajoint/migrate.py

@@ -25,7 +25,7 @@ def migrate_dj011_external_blob_storage_to_dj012(migration_schema, store):
 Proceed?
             """, default='no') == 'yes'
     if do_migration:
-        _migrate_dj011_blob(dj.schema(migration_schema), store)
+        _migrate_dj011_blob(dj.Schema(migration_schema), store)
         print('Migration completed for schema: {}, store: {}.'.format(
                 migration_schema, store))
         return

datajoint/plugin.py

@@ -1,16 +1,8 @@
+from .settings import config
 import pkg_resources
 from pathlib import Path
 from cryptography.exceptions import InvalidSignature
 from setuptools_certificate import hash_pkg, verify
-from .settings import config
-
-discovered_plugins = {
-    entry_point.module_name: dict(plugon=entry_point.name, verified=False)
-    for entry_point
-    in pkg_resources.iter_entry_points('datajoint.plugins')
-    if 'plugin' not in config or entry_point.name not in config['plugin'] or
-    entry_point.module_name in config['plugin'][entry_point.name]
-}
 
 
 def _update_error_stack(plugin_name):
@@ -23,34 +15,24 @@ def _update_error_stack(plugin_name):
         signature = plugin_meta.get_metadata('{}.sig'.format(plugin_name))
         pubkey_path = str(Path(base_meta.egg_info, '{}.pub'.format(base_name)))
         verify(pubkey_path, data, signature)
-        discovered_plugins[plugin_name]['verified'] = True
-        print('DataJoint verified plugin `{}` introduced.'.format(plugin_name))
+        print('DataJoint verified plugin `{}` detected.'.format(plugin_name))
+        return True
     except (FileNotFoundError, InvalidSignature):
-        print('Unverified plugin `{}` introduced.'.format(plugin_name))
+        print('Unverified plugin `{}` detected.'.format(plugin_name))
+        return False
+
+
+def _import_plugins(category):
+    return {
+            entry_point.name: dict(object=entry_point,
+                                    verified=_update_error_stack(
+                                        entry_point.module_name.split('.')[0]))
+            for entry_point
+            in pkg_resources.iter_entry_points('datajoint_plugins.{}'.format(category))
+            if 'plugin' not in config or category not in config['plugin'] or
+            entry_point.module_name.split('.')[0] in config['plugin'][category]
+        }
 
 
-def override(plugin_type, context, method_list=None):
-    relevant_plugins = {
-        k: v for k, v in discovered_plugins.items() if v['plugon'] == plugin_type}
-    if relevant_plugins:
-        for module_name in relevant_plugins:
-            # import plugin
-            module = __import__(module_name)
-            module_dict = module.__dict__
-            # update error stack (if applicable)
-            _update_error_stack(module.__name__)
-            # override based on plugon preference
-            if method_list is not None:
-                new_methods = []
-                for v in method_list:
-                    try:
-                        new_methods.append(getattr(module, v))
-                    except AttributeError:
-                        pass
-                context.update(dict(zip(method_list, new_methods)))
-            else:
-                try:
-                    new_methods = module.__all__
-                except AttributeError:
-                    new_methods = [name for name in module_dict if not name.startswith('_')]
-                context.update({name: module_dict[name] for name in new_methods})
+connection_plugins = _import_plugins('connection')
+type_plugins = _import_plugins('datatype')

datajoint/schema.py datajoint/schemas.pydatajoint/schema.py renamed to datajoint/schemas.py

@@ -16,7 +16,6 @@
 from .user_tables import Part, Computed, Imported, Manual, Lookup
 from .table import lookup_class_name, Log, FreeTable
 import types
-from .plugin import override
 
 logger = logging.getLogger(__name__)
 
@@ -281,8 +280,8 @@ def repl(s):
         body = '\n\n\n'.join(make_class_definition(table) for table in diagram.topological_sort())
         python_code = '\n\n\n'.join((
             '"""This module was auto-generated by datajoint from an existing schema"""',
-            "import datajoint as dj\n\nschema = dj.schema('{db}')".format(db=db),
-            '\n'.join("{module} = dj.create_virtual_module('{module}', '{schema_name}')".format(module=v, schema_name=k)
+            "import datajoint as dj\n\nschema = dj.Schema('{db}')".format(db=db),
+            '\n'.join("{module} = dj.VirtualModule('{module}', '{schema_name}')".format(module=v, schema_name=k)
                       for k, v in module_lookup.items()), body))
         if python_filename is None:
             return python_code
@@ -291,29 +290,30 @@ def repl(s):
                 f.write(python_code)
 
 
-def create_virtual_module(module_name, schema_name, *,
-                          create_schema=False, create_tables=False, connection=None, add_objects=None):
+class VirtualModule(types.ModuleType):
     """
-    Creates a python module with the given name from the name of a schema on the server and
-    automatically adds classes to it corresponding to the tables in the schema.
-    :param module_name: displayed module name
-    :param schema_name: name of the database in mysql
-    :param create_schema: if True, create the schema on the database server
-    :param create_tables: if True, module.schema can be used as the decorator for declaring new
-    :param connection: a dj.Connection object to pass into the schema
-    :param add_objects: additional objects to add to the module
-    :return: the python module containing classes from the schema object and the table classes
+    A virtual module which will contain context for schema.
     """
-    module = types.ModuleType(module_name)
-    _schema = Schema(schema_name, create_schema=create_schema, create_tables=create_tables, connection=connection)
-    if add_objects:
-        module.__dict__.update(add_objects)
-    module.__dict__['schema'] = _schema
-    _schema.spawn_missing_classes(context=module.__dict__)
-    return module
-
-
-override('schema', globals())
+    def __init__(self, module_name, schema_name, *, create_schema=False, 
+                    create_tables=False, connection=None, add_objects=None):
+        """
+        Creates a python module with the given name from the name of a schema on the server and
+        automatically adds classes to it corresponding to the tables in the schema.
+        :param module_name: displayed module name
+        :param schema_name: name of the database in mysql
+        :param create_schema: if True, create the schema on the database server
+        :param create_tables: if True, module.schema can be used as the decorator for declaring new
+        :param connection: a dj.Connection object to pass into the schema
+        :param add_objects: additional objects to add to the module
+        :return: the python module containing classes from the schema object and the table classes
+        """
+        super(VirtualModule, self).__init__(name=module_name)
+        _schema = Schema(schema_name, create_schema=create_schema, create_tables=create_tables,
+                            connection=connection)
+        if add_objects:
+            self.__dict__.update(add_objects)
+        self.__dict__['schema'] = _schema
+        _schema.spawn_missing_classes(context=self.__dict__)
 
 
 def list_schemas(connection=None):

datajoint/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.12.5"
+__version__ = "0.12.5plug"
 
 assert len(__version__) <= 10  # The log table limits version to the 10 characters