Merge pull request #868 from dimitri-yatsenko/cascade-delete

Adding plugin support

Author: guzman-raphael

.github/workflows/development.yaml

@@ -21,8 +21,6 @@ jobs:
             mysql_ver: "5.7"
           - py_ver: "3.6"
             mysql_ver: "5.7"
-          - py_ver: "3.5"
-            mysql_ver: "5.7"
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{matrix.py_ver}}
@@ -50,4 +48,4 @@ jobs:
       - name: Run style tests
         run: |
           flake8 --ignore=E121,E123,E126,E226,E24,E704,W503,W504,E722,F401,W605 datajoint \
-                 --count --max-complexity=62 --max-line-length=127 --statistics
+                 --count --max-complexity=62 --max-line-length=127 --statistics

CHANGELOG.md

@@ -1,13 +1,17 @@
 ## Release notes
 
-### 0.13.0 -- Jan 11, 2020
+### 0.13.0 -- Feb 15, 2021
 * Re-implement query transpilation into SQL, fixing issues (#386, #449, #450, #484). PR #754
 * Re-implement cascading deletes for better performance. PR #839.
-* Add table method `.update1` to update an existing row in its table.
+* Add table method `.update1` to update a row in the table with new values PR #763
 * Python datatypes are now enabled by default in blobs (#761). PR #785
 * Added permissive join and restriction operators `@` and `^` (#785) PR #754
+* Support DataJoint datatype and connection plugins (#715, #729) PR 730, #735
+* add `dj.key_hash` alias to `dj.hash.key_hash`
+* default enable_python_native_blobs to True
+* Drop support for Python 3.5
 
-### 0.12.8 -- Dec 22, 2020
+### 0.12.8 -- Jan 12, 2021
 * table.children, .parents, .descendents, and ancestors can return queryable objects. PR #833
 * Load dependencies before querying dependencies. (#179) PR #833
 * Fix display of part tables in `schema.save`. (#821) PR #833

README.md

@@ -32,9 +32,10 @@ Some Python datatypes such as dicts were coerced into numpy recarrays and then f
 However, since some Python types were coerced into MATLAB types, old blobs and new blobs may now be fetched as different types of objects even if they were inserted the same way. 
 For example, new `dict` objects will be returned as `dict` while the same types of objects inserted with `datajoint 0.11` will be recarrays.
 
-Since this is a big change, we chose to disable full blob support by default as a temporary precaution, which will be removed in version 0.13.
+Since this is a big change, we chose to temporarily disable this feature by default in DataJoint for Python 0.12.x, allowing users to adjust their code if necessary. 
+From 13.x, the flag will default to True (on), and will ultimately be removed when corresponding decode support for the new format is added to datajoint-matlab (see: datajoint-matlab #222, datajoint-python #765).
 
-You may enable it by setting the `enable_python_native_blobs` flag in `dj.config`. 
+The flag is configured by setting the `enable_python_native_blobs` flag in `dj.config`. 
 
 ```python
 import datajoint as dj
@@ -68,7 +69,7 @@ as structured arrays, whereas new record inserted in DataJoint 0.12 with
 appropriate native python type (dict, etc).  
 Furthermore, DataJoint for MATLAB does not yet support unpacking native Python datatypes.
 
-With `dj.config["enable_python_native_blobs"]` set to `False` (default), 
+With `dj.config["enable_python_native_blobs"]` set to `False`, 
 any attempt to insert any datatype other than a numpy array will result in an exception.
 This is meant to get users to read this message in order to allow proper testing
 and migration of pre-0.12 pipelines to 0.12 in a safe manner.

datajoint.pub

@@ -0,0 +1,6 @@
+-----BEGIN PUBLIC KEY-----
+MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDUMOo2U7YQ1uOrKU/IreM3AQP2
+AXJC3au+S9W+dilxHcJ3e98bRVqrFeOofcGeRPoNc38fiLmLDUiBskJeVrpm29Wo
+AkH6yhZWk1o8NvGMhK4DLsJYlsH6tZuOx9NITKzJuOOH6X1I5Ucs7NOSKnmu7g5g
+WTT5kCgF5QAe5JN8WQIDAQAB
+-----END PUBLIC KEY-----

datajoint/__init__.py

@@ -24,7 +24,7 @@
            'Not', 'AndList', 'U', 'Diagram', 'Di', 'ERD',
            'set_password', 'kill',
            'MatCell', 'MatStruct', 'AttributeAdapter',
-           'errors', 'DataJointError', 'key']
+           'errors', 'DataJointError', 'key', 'key_hash']
 
 from .version import __version__
 from .settings import config
@@ -38,6 +38,7 @@
 from .admin import set_password, kill
 from .blob import MatCell, MatStruct
 from .fetch import key
+from .hash import key_hash
 from .attribute_adapter import AttributeAdapter
 from . import errors
 from .errors import DataJointError

datajoint/attribute_adapter.py

@@ -1,5 +1,6 @@
 import re
 from .errors import DataJointError, _support_adapted_types
+from .plugin import type_plugins
 
 
 class AttributeAdapter:
@@ -38,10 +39,11 @@ def get_adapter(context, adapter_name):
         raise DataJointError('Support for Adapted Attribute types is disabled.')
     adapter_name = adapter_name.lstrip('<').rstrip('>')
     try:
-        adapter = context[adapter_name]
+        adapter = (context[adapter_name] if adapter_name in context
+                   else type_plugins[adapter_name]['object'].load())
     except KeyError:
         raise DataJointError(
-            "Attribute adapter '{adapter_name}' is not defined.".format(adapter_name=adapter_name)) from None
+            "Attribute adapter '{adapter_name}' is not defined.".format(adapter_name=adapter_name))
     if not isinstance(adapter, AttributeAdapter):
         raise DataJointError(
             "Attribute adapter '{adapter_name}' must be an instance of datajoint.AttributeAdapter".format(

datajoint/autopopulate.py

@@ -40,7 +40,7 @@ def _rename_attributes(table, props):
             parents = self.target.parents(primary=True, as_objects=True, foreign_key_info=True)
             if not parents:
                 raise DataJointError(
-                    'A relation must have primary dependencies for auto-populate to work') from None
+                    'A relation must have primary dependencies for auto-populate to work')
             self._key_source = _rename_attributes(*parents[0])
             for q in parents[1:]:
                 self._key_source *= _rename_attributes(*q)

datajoint/blob.py

@@ -11,11 +11,10 @@
 import uuid
 import numpy as np
 from .errors import DataJointError
-from .utils import OrderedDict
 from .settings import config
 
 
-mxClassID = OrderedDict((
+mxClassID = dict((
     # see http://www.mathworks.com/help/techdoc/apiref/mxclassid.html
     ('mxUNKNOWN_CLASS', None),
     ('mxCELL_CLASS', None),
@@ -346,8 +345,8 @@ def pack_set(self, t):
             len_u64(it) + it for it in (self.pack_blob(i) for i in t))
 
     def read_dict(self):
-        return OrderedDict((self.read_blob(self.read_value()), self.read_blob(self.read_value()))
-                           for _ in range(self.read_value()))
+        return dict((self.read_blob(self.read_value()), self.read_blob(self.read_value()))
+                    for _ in range(self.read_value()))
 
     def pack_dict(self, d):
         return b"\4" + len_u64(d) + b"".join(

datajoint/connection.py

@@ -15,11 +15,36 @@
 from .dependencies import Dependencies
 from .blob import pack, unpack
 from .hash import uuid_from_buffer
+from .plugin import connection_plugins
 
 logger = logging.getLogger(__name__)
 query_log_max_length = 300
 
 
+def get_host_hook(host_input):
+    if '://' in host_input:
+        plugin_name = host_input.split('://')[0]
+        try:
+            return connection_plugins[plugin_name]['object'].load().get_host(host_input)
+        except KeyError:
+            raise errors.DataJointError(
+                "Connection plugin '{}' not found.".format(plugin_name))
+    else:
+        return host_input
+
+
+def connect_host_hook(connection_obj):
+    if '://' in connection_obj.conn_info['host_input']:
+        plugin_name = connection_obj.conn_info['host_input'].split('://')[0]
+        try:
+            connection_plugins[plugin_name]['object'].load().connect_host(connection_obj)
+        except KeyError:
+            raise errors.DataJointError(
+                "Connection plugin '{}' not found.".format(plugin_name))
+    else:
+        connection_obj.connect()
+
+
 def translate_query_error(client_error, query):
     """
     Take client error and original query and return the corresponding DataJoint exception.
@@ -82,7 +107,8 @@ def conn(host=None, user=None, password=None, *, init_fun=None, reset=False, use
                         #encrypted-connection-options).
     """
     if not hasattr(conn, 'connection') or reset:
-        host = host if host is not None else config['database.host']
+        host_input = host if host is not None else config['database.host']
+        host = get_host_hook(host_input)
         user = user if user is not None else config['database.user']
         password = password if password is not None else config['database.password']
         if user is None:  # pragma: no cover
@@ -91,7 +117,8 @@ def conn(host=None, user=None, password=None, *, init_fun=None, reset=False, use
             password = getpass(prompt="Please enter DataJoint password: ")
         init_fun = init_fun if init_fun is not None else config['connection.init_function']
         use_tls = use_tls if use_tls is not None else config['database.use_tls']
-        conn.connection = Connection(host, user, password, None, init_fun, use_tls)
+        conn.connection = Connection(host, user, password, None, init_fun, use_tls,
+                                     host_input=host_input)
     return conn.connection
 
 
@@ -113,6 +140,10 @@ def fetchall(self):
     def fetchone(self):
         return next(self._iter)
 
+    @property
+    def rowcount(self):
+        return len(self._data)
+
 
 class Connection:
     """
@@ -128,7 +159,9 @@ class Connection:
     :param init_fun: connection initialization function (SQL)
     :param use_tls: TLS encryption option
     """
-    def __init__(self, host, user, password, port=None, init_fun=None, use_tls=None):
+
+    def __init__(self, host, user, password, port=None, init_fun=None, use_tls=None,
+                 host_input=None):
         if ':' in host:
             # the port in the hostname overrides the port argument
             host, port = host.split(':')
@@ -139,11 +172,12 @@ def __init__(self, host, user, password, port=None, init_fun=None, use_tls=None)
         if use_tls is not False:
             self.conn_info['ssl'] = use_tls if isinstance(use_tls, dict) else {'ssl': {}}
         self.conn_info['ssl_input'] = use_tls
+        self.conn_info['host_input'] = host_input
         self.init_fun = init_fun
         print("Connecting {user}@{host}:{port}".format(**self.conn_info))
         self._conn = None
         self._query_cache = None
-        self.connect()
+        connect_host_hook(self)
         if self.is_connected:
             logger.info("Connected {user}@{host}:{port}".format(**self.conn_info))
             self.connection_id = self.query('SELECT connection_id()').fetchone()[0]
@@ -172,15 +206,15 @@ def connect(self):
                              "STRICT_ALL_TABLES,NO_ENGINE_SUBSTITUTION",
                     charset=config['connection.charset'],
                     **{k: v for k, v in self.conn_info.items()
-                       if k != 'ssl_input'})
+                       if k not in ['ssl_input', 'host_input']})
             except client.err.InternalError:
                 self._conn = client.connect(
                     init_command=self.init_fun,
                     sql_mode="NO_ZERO_DATE,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO,"
                              "STRICT_ALL_TABLES,NO_ENGINE_SUBSTITUTION",
                     charset=config['connection.charset'],
                     **{k: v for k, v in self.conn_info.items()
-                       if not(k == 'ssl_input' or
+                       if not(k in ['ssl_input', 'host_input'] or
                               k == 'ssl' and self.conn_info['ssl_input'] is None)})
         self._conn.autocommit(True)
 
@@ -223,7 +257,7 @@ def _execute_query(cursor, query, args, suppress_warnings):
                     warnings.simplefilter("ignore")
                 cursor.execute(query, args)
         except client.err.Error as err:
-            raise translate_query_error(err, query) from None
+            raise translate_query_error(err, query)
 
     def query(self, query, args=(), *, as_dict=False, suppress_warnings=True, reconnect=None):
         """
@@ -247,7 +281,7 @@ def query(self, query, args=(), *, as_dict=False, suppress_warnings=True, reconn
             try:
                 buffer = cache_path.read_bytes()
             except FileNotFoundError:
-                pass   # proceed to the normal query
+                pass   # proceed to query the database
             else:
                 return EmulatedCursor(unpack(buffer))
 
@@ -262,10 +296,10 @@ def query(self, query, args=(), *, as_dict=False, suppress_warnings=True, reconn
             if not reconnect:
                 raise
             warnings.warn("MySQL server has gone away. Reconnecting to the server.")
-            self.connect()
+            connect_host_hook(self)
             if self._in_transaction:
                 self.cancel_transaction()
-                raise errors.LostConnectionError("Connection was lost during a transaction.") from None
+                raise errors.LostConnectionError("Connection was lost during a transaction.")
             logger.debug("Re-executing")
             cursor = self._conn.cursor(cursor=cursor_class)
             self._execute_query(cursor, query, args, suppress_warnings)

datajoint/declare.py

@@ -9,8 +9,6 @@
 from .errors import DataJointError, _support_filepath_types, FILEPATH_FEATURE_SWITCH
 from .attribute_adapter import get_adapter
 
-from .utils import OrderedDict
-
 UUID_DATA_TYPE = 'binary(16)'
 MAX_TABLE_NAME_LENGTH = 64
 CONSTANT_LITERALS = {'CURRENT_TIMESTAMP'}  # SQL literals to be used without quotes (case insensitive)
@@ -46,7 +44,7 @@ def match_type(attribute_type):
     try:
         return next(category for category, pattern in TYPE_PATTERN.items() if pattern.match(attribute_type))
     except StopIteration:
-        raise DataJointError("Unsupported attribute type {type}".format(type=attribute_type)) from None
+        raise DataJointError("Unsupported attribute type {type}".format(type=attribute_type))
 
 
 logger = logging.getLogger(__name__)
@@ -136,7 +134,7 @@ def compile_foreign_key(line, context, attributes, primary_key, attr_sql, foreig
         try:
             result = foreign_key_parser_old.parseString(line)
         except pp.ParseBaseException as err:
-            raise DataJointError('Parsing error in line "%s". %s.' % (line, err)) from None
+            raise DataJointError('Parsing error in line "%s". %s.' % (line, err))
         else:
             obsolete = True
     try:
@@ -303,7 +301,7 @@ def _make_attribute_alter(new, old, primary_key):
     name_regexp = re.compile(r"^`(?P<name>\w+)`")
     original_regexp = re.compile(r'COMMENT "{\s*(?P<name>\w+)\s*}')
     matched = ((name_regexp.match(d), original_regexp.search(d)) for d in new)
-    new_names = OrderedDict((d.group('name'), n and n.group('name')) for d, n in matched)
+    new_names = dict((d.group('name'), n and n.group('name')) for d, n in matched)
     old_names = [name_regexp.search(d).group('name') for d in old]
 
     # verify that original names are only used once
@@ -433,7 +431,7 @@ def compile_attribute(line, in_key, foreign_key_sql, context):
         match = attribute_parser.parseString(line + '#', parseAll=True)
     except pp.ParseException as err:
         raise DataJointError('Declaration error in position {pos} in line:\n  {line}\n{msg}'.format(
-            line=err.args[0], pos=err.args[1], msg=err.args[2])) from None
+            line=err.args[0], pos=err.args[1], msg=err.args[2]))
     match['comment'] = match['comment'].rstrip('#')
     if 'default' not in match:
         match['default'] = ''