Merge pull request #822 from datajoint/master

Bringing Release013 in sync with master

Author: guzman-raphael

.gitignore

@@ -23,4 +23,5 @@ docker-compose.yml
 notebook
 .vscode
 __main__.py
-jupyter_custom.js
+jupyter_custom.js
+apk_requirements.txt

LNX-docker-compose.yml

@@ -1,3 +1,4 @@
+# docker-compose -f LNX-docker-compose.yml --env-file LNX.env up --build --exit-code-from app
 version: '2.2'
 x-net: &net
   networks:
@@ -31,7 +32,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: raphaelguzman/nginx:v0.0.6
+    image: raphaelguzman/nginx:v0.0.10
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306

datajoint/admin.py

@@ -49,10 +49,11 @@ def kill(restriction=None, connection=None, order_by=None):  # pragma: no cover
     while True:
         print('  ID USER         HOST          STATE         TIME    INFO')
         print('+--+ +----------+ +-----------+ +-----------+ +-----+')
-        cur = connection.query(query, as_dict=True)
+        cur = ({k.lower(): v for k, v in elem.items()}
+               for elem in connection.query(query, as_dict=True))
         for process in cur:
             try:
-                print('{ID:>4d} {USER:<12s} {HOST:<12s} {STATE:<12s} {TIME:>7d}  {INFO}'.format(**process))
+                print('{id:>4d} {user:<12s} {host:<12s} {state:<12s} {time:>7d}  {info}'.format(**process))
             except TypeError:
                 print(process)
         response = input('process to kill or "q" to quit > ')
@@ -88,9 +89,10 @@ def kill_quick(restriction=None, connection=None):
     query = 'SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()' + (
         "" if restriction is None else ' AND (%s)' % restriction)
 
-    cur = connection.query(query, as_dict=True)
+    cur = ({k.lower(): v for k, v in elem.items()}
+           for elem in connection.query(query, as_dict=True))
     nkill = 0
     for process in cur:
-        connection.query('kill %d' % process['ID'])
+        connection.query('kill %d' % process['id'])
         nkill += 1
     return nkill

datajoint/connection.py

@@ -12,8 +12,8 @@
 from . import errors
 from .dependencies import Dependencies
 
-# client errors to catch
-client_errors = (client.err.InterfaceError, client.err.DatabaseError)
+logger = logging.getLogger(__name__)
+query_log_max_length = 300
 
 
 def translate_query_error(client_error, query):
@@ -23,39 +23,39 @@ def translate_query_error(client_error, query):
     :param query: sql query with placeholders
     :return: an instance of the corresponding subclass of datajoint.errors.DataJointError
     """
+    logger.debug('type: {}, args: {}'.format(type(client_error), client_error.args))
+
+    err, *args = client_error.args
+
     # Loss of connection errors
-    if isinstance(client_error, client.err.InterfaceError) and client_error.args[0] == "(0, '')":
-        return errors.LostConnectionError('Server connection lost due to an interface error.', *client_error.args[1:])
-    disconnect_codes = {
-        2006: "Connection timed out",
-        2013: "Server connection lost"}
-    if isinstance(client_error, client.err.OperationalError) and client_error.args[0] in disconnect_codes:
-        return errors.LostConnectionError(disconnect_codes[client_error.args[0]], *client_error.args[1:])
+    if err in (0, "(0, '')"):
+        return errors.LostConnectionError('Server connection lost due to an interface error.', *args)
+    if err == 2006:
+        return errors.LostConnectionError("Connection timed out", *args)
+    if err == 2013:
+        return errors.LostConnectionError("Server connection lost", *args)
     # Access errors
-    if isinstance(client_error, client.err.OperationalError) and client_error.args[0] in (1044, 1142):
-        return errors.AccessError('Insufficient privileges.', client_error.args[1],  query)
+    if err in (1044, 1142):
+        return errors.AccessError('Insufficient privileges.', args[0],  query)
     # Integrity errors
-    if isinstance(client_error, client.err.IntegrityError) and client_error.args[0] == 1062:
-        return errors.DuplicateError(*client_error.args[1:])
-    if isinstance(client_error, client.err.IntegrityError) and client_error.args[0] == 1452:
-        return errors.IntegrityError(*client_error.args[1:])
+    if err == 1062:
+        return errors.DuplicateError(*args)
+    if err == 1452:
+        return errors.IntegrityError(*args)
     # Syntax errors
-    if isinstance(client_error, client.err.ProgrammingError) and client_error.args[0] == 1064:
-        return errors.QuerySyntaxError(client_error.args[1], query)
+    if err == 1064:
+        return errors.QuerySyntaxError(args[0], query)
     # Existence errors
-    if isinstance(client_error, client.err.ProgrammingError) and client_error.args[0] == 1146:
-        return errors.MissingTableError(client_error.args[1], query)
-    if isinstance(client_error, client.err.InternalError) and client_error.args[0] == 1364:
-        return errors.MissingAttributeError(*client_error.args[1:])
-    if isinstance(client_error, client.err.InternalError) and client_error.args[0] == 1054:
-        return errors.UnknownAttributeError(*client_error.args[1:])
+    if err == 1146:
+        return errors.MissingTableError(args[0], query)
+    if err == 1364:
+        return errors.MissingAttributeError(*args)
+    if err == 1054:
+        return errors.UnknownAttributeError(*args)
     # all the other errors are re-raised in original form
     return client_error
 
 
-logger = logging.getLogger(__name__)
-
-
 def conn(host=None, user=None, password=None, *, init_fun=None, reset=False, use_tls=None):
     """
     Returns a persistent connection object to be shared by multiple modules.
@@ -192,7 +192,7 @@ def _execute_query(cursor, query, args, cursor_class, suppress_warnings):
                     # suppress all warnings arising from underlying SQL library
                     warnings.simplefilter("ignore")
                 cursor.execute(query, args)
-        except client_errors as err:
+        except client.err.Error as err:
             raise translate_query_error(err, query) from None
 
     def query(self, query, args=(), *, as_dict=False, suppress_warnings=True, reconnect=None):
@@ -207,7 +207,7 @@ def query(self, query, args=(), *, as_dict=False, suppress_warnings=True, reconn
         """
         if reconnect is None:
             reconnect = config['database.reconnect']
-        logger.debug("Executing SQL:" + query[0:300])
+        logger.debug("Executing SQL:" + query[:query_log_max_length])
         cursor_class = client.cursors.DictCursor if as_dict else client.cursors.Cursor
         cursor = self._conn.cursor(cursor=cursor_class)
         try:

datajoint/dependencies.py

@@ -43,15 +43,15 @@ def load(self):
             self.add_node(n, primary_key=pk)
 
         # load foreign keys
-        keys = self._conn.query("""
+        keys = ({k.lower(): v for k, v in elem.items()} for elem in self._conn.query("""
         SELECT constraint_name,
             concat('`', table_schema, '`.`', table_name, '`') as referencing_table,
             concat('`', referenced_table_schema, '`.`',  referenced_table_name, '`') as referenced_table,
             column_name, referenced_column_name
         FROM information_schema.key_column_usage
         WHERE referenced_table_name NOT LIKE "~%%" AND (referenced_table_schema in ('{schemas}') OR
             referenced_table_schema is not NULL AND table_schema in ('{schemas}'))
-        """.format(schemas="','".join(self._conn.schemas)), as_dict=True)
+        """.format(schemas="','".join(self._conn.schemas)), as_dict=True))
         fks = defaultdict(lambda: dict(attr_map=OrderedDict()))
         for key in keys:
             d = fks[(key['constraint_name'], key['referencing_table'], key['referenced_table'])]

datajoint/external.py

@@ -276,11 +276,11 @@ def references(self):
         """
         :return: generator of referencing table names and their referencing columns
         """
-        return self.connection.query("""
+        return ({k.lower(): v for k, v in elem.items()} for elem in self.connection.query("""
         SELECT concat('`', table_schema, '`.`', table_name, '`') as referencing_table, column_name
         FROM information_schema.key_column_usage
         WHERE referenced_table_name="{tab}" and referenced_table_schema="{db}"
-        """.format(tab=self.table_name, db=self.database), as_dict=True)
+        """.format(tab=self.table_name, db=self.database), as_dict=True))
 
     def fetch_external_paths(self, **fetch_kwargs):
         """

datajoint/migrate.py

@@ -42,14 +42,14 @@ def _migrate_dj011_blob(schema, default_store):
         '`{db}`.`~external`'.format(db=schema.database))
 
     # get referencing tables
-    refs = query("""
+    refs = [{k.lower(): v for k, v in elem.items()} for elem in query("""
     SELECT concat('`', table_schema, '`.`', table_name, '`')
             as referencing_table, column_name, constraint_name
     FROM information_schema.key_column_usage
     WHERE referenced_table_name="{tab}" and referenced_table_schema="{db}"
     """.format(
         tab=legacy_external.table_name,
-        db=legacy_external.database), as_dict=True).fetchall()
+        db=legacy_external.database), as_dict=True).fetchall()]
 
     for ref in refs:
         # get comment
@@ -142,14 +142,14 @@ def _migrate_dj011_blob(schema, default_store):
 
     # Drop the old external table but make sure it's no longer referenced
     # get referencing tables
-    refs = query("""
+    refs = [{k.lower(): v for k, v in elem.items()} for elem in query("""
     SELECT concat('`', table_schema, '`.`', table_name, '`') as
         referencing_table, column_name, constraint_name
     FROM information_schema.key_column_usage
     WHERE referenced_table_name="{tab}" and referenced_table_schema="{db}"
     """.format(
         tab=legacy_external.table_name,
-        db=legacy_external.database), as_dict=True).fetchall()
+        db=legacy_external.database), as_dict=True).fetchall()]
 
     assert not refs, 'Some references still exist'
 

datajoint/schemas.py

@@ -73,7 +73,6 @@ def __init__(self, schema_name, context=None, *, connection=None, create_schema=
                 logger.info("Creating schema `{name}`.".format(name=schema_name))
                 try:
                     connection.query("CREATE DATABASE `{name}`".format(name=schema_name))
-                    logger.info('Creating schema `{name}`.'.format(name=schema_name))
                 except pymysql.OperationalError:
                     raise DataJointError(
                         "Schema `{name}` does not exist and could not be created. "

local-docker-compose.yml

@@ -1,3 +1,4 @@
+# docker-compose -f local-docker-compose.yml --env-file LNX.env up --build
 version: '2.2'
 x-net: &net
   networks:
@@ -33,7 +34,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: raphaelguzman/nginx:v0.0.6
+    image: raphaelguzman/nginx:v0.0.10
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306
@@ -106,4 +107,4 @@ services:
       # - ./notebook:/home/dja/notebooks
       # - ../dj-python-101/ch1:/home/dja/tutorials
 networks:
-  main:
+  main:

tests/test_schema.py

@@ -68,7 +68,7 @@ def test_unauthorized_database():
     """
     an attempt to create a database to which user has no privileges should raise an informative exception.
     """
-    dj.Schema('unauthorized_schema', connection=dj.conn(**CONN_INFO))
+    dj.Schema('unauthorized_schema', connection=dj.conn(reset=True, **CONN_INFO))
 
 
 def test_drop_database():