Merge remote-tracking branch 'upstream/master' into issue-666

Author: Chris Turner

LNX-docker-compose.yml

@@ -19,27 +19,27 @@ services:
       - MINIO_ACCESS_KEY=datajoint
       - MINIO_SECRET_KEY=datajoint
     # ports:
-    #   - "9000:9000"
+    #   - "80:80"
     # volumes:
     #   - ./minio/config:/root/.minio
     #   - ./minio/data:/data
-    command: server /data
+    command: server --address ":80" /data
     healthcheck:
-      test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"]
+      test: ["CMD", "curl", "--fail", "http://minio:80/minio/health/live"]
       timeout: 5s
       retries: 60
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: raphaelguzman/nginx:v0.0.4
+    image: raphaelguzman/nginx:v0.0.5
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306
     - ADD_minio_TYPE=MINIO
-    - ADD_minio_ENDPOINT=minio:9000
+    - ADD_minio_ENDPOINT=minio:80
     - ADD_minio_PREFIX=/
     # ports:
-    #   - "9000:9000"
+    #   - "80:80"
     #   - "443:443"
     #   - "3306:3306"
     depends_on:
@@ -60,7 +60,7 @@ services:
       - DJ_TEST_HOST=fakeservices.datajoint.io
       - DJ_TEST_USER=datajoint
       - DJ_TEST_PASSWORD=datajoint
-      - S3_ENDPOINT=fakeservices.datajoint.io:9000
+      - S3_ENDPOINT=fakeservices.datajoint.io
       - S3_ACCESS_KEY=datajoint
       - S3_SECRET_KEY=datajoint
       - S3_BUCKET=datajoint-test

datajoint/admin.py

@@ -21,11 +21,12 @@ def set_password(new_password=None, connection=None, update_config=None):   # pr
         config.save_local(verbose=True)
 
 
-def kill(restriction=None, connection=None):  # pragma: no cover
+def kill(restriction=None, connection=None, order_by=None):  # pragma: no cover
     """
     view and kill database connections.
     :param restriction: restriction to be applied to processlist
     :param connection: a datajoint.Connection object. Default calls datajoint.conn()
+    :param order_by: order by string clause for output ordering. defaults to 'id'.
 
     Restrictions are specified as strings and can involve any of the attributes of
     information_schema.processlist: ID, USER, HOST, DB, COMMAND, TIME, STATE, INFO.
@@ -39,7 +40,8 @@ def kill(restriction=None, connection=None):  # pragma: no cover
         connection = conn()
 
     query = 'SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()' + (
-        "" if restriction is None else ' AND (%s)' % restriction)
+        "" if restriction is None else ' AND (%s)' % restriction) + (
+            ' ORDER BY %s' % (order_by or 'id'))
 
     while True:
         print('  ID USER         HOST          STATE         TIME    INFO')

datajoint/connection.py

@@ -185,7 +185,7 @@ def is_connected(self):
         return True
 
     @staticmethod
-    def __execute_query(cursor, query, args, cursor_class, suppress_warnings):
+    def _execute_query(cursor, query, args, cursor_class, suppress_warnings):
         try:
             with warnings.catch_warnings():
                 if suppress_warnings:
@@ -211,7 +211,7 @@ def query(self, query, args=(), *, as_dict=False, suppress_warnings=True, reconn
         cursor_class = client.cursors.DictCursor if as_dict else client.cursors.Cursor
         cursor = self._conn.cursor(cursor=cursor_class)
         try:
-            self.__execute_query(cursor, query, args, cursor_class, suppress_warnings)
+            self._execute_query(cursor, query, args, cursor_class, suppress_warnings)
         except errors.LostConnectionError:
             if not reconnect:
                 raise
@@ -222,7 +222,7 @@ def query(self, query, args=(), *, as_dict=False, suppress_warnings=True, reconn
                 raise errors.LostConnectionError("Connection was lost during a transaction.") from None
             logger.debug("Re-executing")
             cursor = self._conn.cursor(cursor=cursor_class)
-            self.__execute_query(cursor, query, args, cursor_class, suppress_warnings)
+            self._execute_query(cursor, query, args, cursor_class, suppress_warnings)
         return cursor
 
     def get_user(self):

datajoint/errors.py

@@ -81,6 +81,12 @@ class MissingExternalFile(DataJointError):
     """
 
 
+class BucketInaccessible(DataJointError):
+    """
+    Error raised when a S3 bucket is inaccessible
+    """
+
+
 # environment variables to control availability of experimental features
 
 ADAPTED_TYPE_SWITCH = "DJ_SUPPORT_ADAPTED_TYPES"

datajoint/external.py

@@ -46,6 +46,9 @@ def __init__(self, connection, store=None, database=None):
         if not self.is_declared:
             self.declare()
         self._s3 = None
+        if self.spec['protocol'] == 'file' and not Path(self.spec['location']).is_dir():
+            raise FileNotFoundError('Inaccessible local directory %s' %
+                                    self.spec['location']) from None
 
     @property
     def definition(self):

datajoint/fetch.py

@@ -173,7 +173,9 @@ def __call__(self, *attrs, offset=None, limit=None, order_by=None, format=None,
             attributes = [a for a in attrs if not is_key(a)]
             ret = self._expression.proj(*attributes).fetch(
                 offset=offset, limit=limit, order_by=order_by,
-                as_dict=False, squeeze=squeeze, download_path=download_path)
+                as_dict=False, squeeze=squeeze, download_path=download_path,
+                format='array'
+            )
             if attrs_as_dict:
                 ret = [{k: v for k, v in zip(ret.dtype.names, x) if k in attrs} for x in ret]
             else:

datajoint/s3.py

@@ -14,11 +14,11 @@ class Folder:
     A Folder instance manipulates a flat folder of objects within an S3-compatible object store
     """
     def __init__(self, endpoint, bucket, access_key, secret_key, *, secure=False, **_):
-        self.client = minio.Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=secure)
+        self.client = minio.Minio(endpoint, access_key=access_key, secret_key=secret_key,
+                                  secure=secure)
         self.bucket = bucket
         if not self.client.bucket_exists(bucket):
-            warnings.warn('Creating bucket "%s"' % self.bucket)
-            self.client.make_bucket(self.bucket)
+            raise errors.BucketInaccessible('Inaccessible s3 bucket %s' % bucket) from None
 
     def put(self, name, buffer):
         return self.client.put_object(
@@ -63,6 +63,6 @@ def get_size(self, name):
 
     def remove_object(self, name):
         try:
-            self.client.remove_objects(self.bucket, str(name))
+            self.client.remove_object(self.bucket, str(name))
         except minio.ResponseError:
             return errors.DataJointError('Failed to delete %s from s3 storage' % name)

datajoint/table.py

@@ -539,7 +539,6 @@ def describe(self, context=None, printout=True):
                             parent_name = list(self.connection.dependencies.in_edges(parent_name))[0][0]
                             lst = [(attr, ref) for attr, ref in fk_props['attr_map'].items() if ref != attr]
                             definition += '->{props} {class_name}.proj({proj_list})\n'.format(
-                                attr_list=', '.join(r[0] for r in lst),
                                 props=index_props,
                                 class_name=lookup_class_name(parent_name, context) or parent_name,
                                 proj_list=','.join('{}="{}"'.format(a, b) for a, b in lst))

docs-parts/computation/06-distributed-computing_kill_order_by.rst

@@ -0,0 +1,14 @@
+
+For example, to sort the output by hostname in descending order:
+
+.. code-block:: text
+
+    In [3]: dj.kill(None, None, 'host desc')
+    Out[3]:
+      ID USER         HOST          STATE         TIME    INFO
+    +--+ +----------+ +-----------+ +-----------+ +-----+
+      33 chris        localhost:54840                 1261  None
+      17 chris        localhost:54587                 3246  None
+       4 event_scheduler localhost    Waiting on empty queue  187180  None
+    process to kill or "q" to quit > q
+

local-docker-compose.yml

@@ -20,28 +20,28 @@ services:
       - MINIO_ACCESS_KEY=datajoint
       - MINIO_SECRET_KEY=datajoint
     # ports:
-    #   - "9000:9000"
+    #   - "80:80"
     # To persist MinIO data and config
     # volumes:
     #   - ./minio/data:/data
     #   - ./minio/config:/root/.minio
-    command: server /data
+    command: server --address ":80" /data
     healthcheck:
-      test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"]
+      test: ["CMD", "curl", "--fail", "http://minio:80/minio/health/live"]
       timeout: 5s
       retries: 60
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: raphaelguzman/nginx:v0.0.4
+    image: raphaelguzman/nginx:v0.0.5
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306
     - ADD_minio_TYPE=MINIO
-    - ADD_minio_ENDPOINT=minio:9000
+    - ADD_minio_ENDPOINT=minio:80
     - ADD_minio_PREFIX=/
     ports:
-      - "9000:9000"
+      - "80:80"
       - "443:443"
       - "3306:3306"
     depends_on:
@@ -63,7 +63,7 @@ services:
       - DJ_TEST_USER=datajoint
       - DJ_TEST_PASSWORD=datajoint
       # If running tests locally, make sure to add entry in /etc/hosts for 127.0.0.1 fakeservices.datajoint.io
-      - S3_ENDPOINT=fakeservices.datajoint.io:9000
+      - S3_ENDPOINT=fakeservices.datajoint.io
       - S3_ACCESS_KEY=datajoint
       - S3_SECRET_KEY=datajoint
       - S3_BUCKET=datajoint-test