Resolve merge conflicts.

Author: guzman-raphael

.gitignore

@@ -23,4 +23,5 @@ docker-compose.yml
 notebook
 .vscode
 __main__.py
-jupyter_custom.js
+jupyter_custom.js
+apk_requirements.txt

CHANGELOG.md

@@ -1,5 +1,10 @@
 ## Release notes
 
+### 0.12.7b1 -- Oct 27, 2020
+* Fix case sensitivity issues to adapt to MySQL 8+.  PR #819
+* Fix pymysql regression bug (#814) PR #816
+* Adapted attribute types now have dtype=object in all recarray results. PR #811 
+
 ### 0.12.6b1 -- May 18, 2020
 * Add `order_by` to `dj.kill` (#668, #779) PR #775, #783
 * Add explicit S3 bucket and file storage location existence checks (#748) PR #781

LNX-docker-compose.yml

@@ -1,3 +1,4 @@
+# docker-compose -f LNX-docker-compose.yml --env-file LNX.env up --build --exit-code-from app
 version: '2.2'
 x-net: &net
   networks:
@@ -31,7 +32,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: raphaelguzman/nginx:v0.0.6
+    image: raphaelguzman/nginx:v0.0.10
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306

README.md

@@ -114,6 +114,7 @@ GID=1000
 * `cp local-docker-compose.yml docker-compose.yml`
 * `docker-compose up -d` (Note configured `JUPYTER_PASSWORD`)
 * Select a means of running Tests e.g. Docker Terminal, or Local Terminal (see bottom)
+* Add entry in `/etc/hosts` for `127.0.0.1 fakeservices.datajoint.io`
 * Run desired tests. Some examples are as follows:
 
 | Use Case                     | Shell Code                                                                    |
@@ -137,4 +138,4 @@ GID=1000
 ### Launch Jupyter Notebook for Interactive Use
 * Navigate to `localhost:8888`
 * Input Jupyter password
-* Launch a notebook i.e. `New > Python 3`
+* Launch a notebook i.e. `New > Python 3`

datajoint/admin.py

@@ -49,10 +49,11 @@ def kill(restriction=None, connection=None, order_by=None):  # pragma: no cover
     while True:
         print('  ID USER         HOST          STATE         TIME    INFO')
         print('+--+ +----------+ +-----------+ +-----------+ +-----+')
-        cur = connection.query(query, as_dict=True)
+        cur = ({k.lower(): v for k, v in elem.items()}
+               for elem in connection.query(query, as_dict=True))
         for process in cur:
             try:
-                print('{ID:>4d} {USER:<12s} {HOST:<12s} {STATE:<12s} {TIME:>7d}  {INFO}'.format(**process))
+                print('{id:>4d} {user:<12s} {host:<12s} {state:<12s} {time:>7d}  {info}'.format(**process))
             except TypeError:
                 print(process)
         response = input('process to kill or "q" to quit > ')
@@ -88,9 +89,10 @@ def kill_quick(restriction=None, connection=None):
     query = 'SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()' + (
         "" if restriction is None else ' AND (%s)' % restriction)
 
-    cur = connection.query(query, as_dict=True)
+    cur = ({k.lower(): v for k, v in elem.items()}
+           for elem in connection.query(query, as_dict=True))
     nkill = 0
     for process in cur:
-        connection.query('kill %d' % process['ID'])
+        connection.query('kill %d' % process['id'])
         nkill += 1
     return nkill

datajoint/attribute_adapter.py

@@ -25,7 +25,7 @@ def get(self, value):
     def put(self, obj):
         """
         convert an object of the adapted type into a value that DataJoint can store in a table attribute
-        :param object: an object of the adapted type
+        :param obj: an object of the adapted type
         :return: value to store in the database
         """
         raise NotImplementedError('Undefined attribute adapter')

datajoint/connection.py

@@ -13,8 +13,8 @@
 from .dependencies import Dependencies
 from .plugin import connection_plugins
 
-# client errors to catch
-client_errors = (client.err.InterfaceError, client.err.DatabaseError)
+logger = logging.getLogger(__name__)
+query_log_max_length = 300
 
 
 def get_host_hook(host_input):
@@ -48,39 +48,39 @@ def translate_query_error(client_error, query):
     :param query: sql query with placeholders
     :return: an instance of the corresponding subclass of datajoint.errors.DataJointError
     """
+    logger.debug('type: {}, args: {}'.format(type(client_error), client_error.args))
+
+    err, *args = client_error.args
+
     # Loss of connection errors
-    if isinstance(client_error, client.err.InterfaceError) and client_error.args[0] == "(0, '')":
-        return errors.LostConnectionError('Server connection lost due to an interface error.', *client_error.args[1:])
-    disconnect_codes = {
-        2006: "Connection timed out",
-        2013: "Server connection lost"}
-    if isinstance(client_error, client.err.OperationalError) and client_error.args[0] in disconnect_codes:
-        return errors.LostConnectionError(disconnect_codes[client_error.args[0]], *client_error.args[1:])
+    if err in (0, "(0, '')"):
+        return errors.LostConnectionError('Server connection lost due to an interface error.', *args)
+    if err == 2006:
+        return errors.LostConnectionError("Connection timed out", *args)
+    if err == 2013:
+        return errors.LostConnectionError("Server connection lost", *args)
     # Access errors
-    if isinstance(client_error, client.err.OperationalError) and client_error.args[0] in (1044, 1142):
-        return errors.AccessError('Insufficient privileges.', client_error.args[1],  query)
+    if err in (1044, 1142):
+        return errors.AccessError('Insufficient privileges.', args[0],  query)
     # Integrity errors
-    if isinstance(client_error, client.err.IntegrityError) and client_error.args[0] == 1062:
-        return errors.DuplicateError(*client_error.args[1:])
-    if isinstance(client_error, client.err.IntegrityError) and client_error.args[0] == 1452:
-        return errors.IntegrityError(*client_error.args[1:])
+    if err == 1062:
+        return errors.DuplicateError(*args)
+    if err == 1452:
+        return errors.IntegrityError(*args)
     # Syntax errors
-    if isinstance(client_error, client.err.ProgrammingError) and client_error.args[0] == 1064:
-        return errors.QuerySyntaxError(client_error.args[1], query)
+    if err == 1064:
+        return errors.QuerySyntaxError(args[0], query)
     # Existence errors
-    if isinstance(client_error, client.err.ProgrammingError) and client_error.args[0] == 1146:
-        return errors.MissingTableError(client_error.args[1], query)
-    if isinstance(client_error, client.err.InternalError) and client_error.args[0] == 1364:
-        return errors.MissingAttributeError(*client_error.args[1:])
-    if isinstance(client_error, client.err.InternalError) and client_error.args[0] == 1054:
-        return errors.UnknownAttributeError(*client_error.args[1:])
+    if err == 1146:
+        return errors.MissingTableError(args[0], query)
+    if err == 1364:
+        return errors.MissingAttributeError(*args)
+    if err == 1054:
+        return errors.UnknownAttributeError(*args)
     # all the other errors are re-raised in original form
     return client_error
 
 
-logger = logging.getLogger(__name__)
-
-
 def conn(host=None, user=None, password=None, *, init_fun=None, reset=False, use_tls=None):
     """
     Returns a persistent connection object to be shared by multiple modules.
@@ -221,7 +221,7 @@ def _execute_query(cursor, query, args, cursor_class, suppress_warnings):
                     # suppress all warnings arising from underlying SQL library
                     warnings.simplefilter("ignore")
                 cursor.execute(query, args)
-        except client_errors as err:
+        except client.err.Error as err:
             raise translate_query_error(err, query)
 
     def query(self, query, args=(), *, as_dict=False, suppress_warnings=True, reconnect=None):
@@ -236,7 +236,7 @@ def query(self, query, args=(), *, as_dict=False, suppress_warnings=True, reconn
         """
         if reconnect is None:
             reconnect = config['database.reconnect']
-        logger.debug("Executing SQL:" + query[0:300])
+        logger.debug("Executing SQL:" + query[:query_log_max_length])
         cursor_class = client.cursors.DictCursor if as_dict else client.cursors.Cursor
         cursor = self._conn.cursor(cursor=cursor_class)
         try:

datajoint/dependencies.py

@@ -8,8 +8,8 @@ class Dependencies(nx.DiGraph):
     """
     The graph of dependencies (foreign keys) between loaded tables.
 
-    Note: the 'connnection' argument should normally be supplied;
-    Empty use is permitted to facilliate use of networkx algorithms which
+    Note: the 'connection' argument should normally be supplied;
+    Empty use is permitted to facilitate use of networkx algorithms which
     internally create objects with the expectation of empty constructors.
     See also: https://github.com/datajoint/datajoint-python/pull/443
     """
@@ -43,15 +43,15 @@ def load(self):
             self.add_node(n, primary_key=pk)
 
         # load foreign keys
-        keys = self._conn.query("""
+        keys = ({k.lower(): v for k, v in elem.items()} for elem in self._conn.query("""
         SELECT constraint_name,
             concat('`', table_schema, '`.`', table_name, '`') as referencing_table,
             concat('`', referenced_table_schema, '`.`',  referenced_table_name, '`') as referenced_table,
             column_name, referenced_column_name
         FROM information_schema.key_column_usage
         WHERE referenced_table_name NOT LIKE "~%%" AND (referenced_table_schema in ('{schemas}') OR
             referenced_table_schema is not NULL AND table_schema in ('{schemas}'))
-        """.format(schemas="','".join(self._conn.schemas)), as_dict=True)
+        """.format(schemas="','".join(self._conn.schemas)), as_dict=True))
         fks = defaultdict(lambda: dict(attr_map=OrderedDict()))
         for key in keys:
             d = fks[(key['constraint_name'], key['referencing_table'], key['referenced_table'])]
@@ -107,7 +107,6 @@ def descendants(self, full_table_name):
         """
         nodes = self.subgraph(
             nx.algorithms.dag.descendants(self, full_table_name))
-
         return [full_table_name] + list(
             nx.algorithms.dag.topological_sort(nodes))
 

datajoint/external.py

@@ -276,11 +276,11 @@ def references(self):
         """
         :return: generator of referencing table names and their referencing columns
         """
-        return self.connection.query("""
+        return ({k.lower(): v for k, v in elem.items()} for elem in self.connection.query("""
         SELECT concat('`', table_schema, '`.`', table_name, '`') as referencing_table, column_name
         FROM information_schema.key_column_usage
         WHERE referenced_table_name="{tab}" and referenced_table_schema="{db}"
-        """.format(tab=self.table_name, db=self.database), as_dict=True)
+        """.format(tab=self.table_name, db=self.database), as_dict=True))
 
     def fetch_external_paths(self, **fetch_kwargs):
         """

datajoint/heading.py

@@ -279,7 +279,7 @@ def init_from_database(self, conn, database, table_name, context):
 
             # fill out dtype. All floats and non-nullable integers are turned into specific dtypes
             attr['dtype'] = object
-            if attr['numeric']:
+            if attr['numeric'] and not attr['adapter']:
                 is_integer = TYPE_PATTERN['INTEGER'].match(attr['type'])
                 is_float = TYPE_PATTERN['FLOAT'].match(attr['type'])
                 if is_integer and not attr['nullable'] or is_float: