Merge pull request #787 from guzman-raphael/plugin

Plugin: Merge in 0.12.6 and prepare for release 0.12.6b1

Author: eywalker

.dockerignore

@@ -0,0 +1,6 @@
+notebook
+build
+*.egg-info
+dist
+.vscode
+__pycache__

.travis.yml

@@ -11,14 +11,20 @@ env:
 services: 
 - docker
 main: &main
-      stage: Alpine
+      stage: "Tests & Coverage: Alpine"
       os: linux
       dist: xenial # precise, trusty, xenial, bionic
       language: shell
       script:
         - docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
 jobs:
   include:
+    - stage: "Lint: Syntax"
+      language: python
+      install:
+        - pip install flake8
+      script:
+        - flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
     - <<: *main
       env:
       - PY_VER: "3.8"
@@ -39,31 +45,15 @@ jobs:
       env:
       - PY_VER: "3.8"
       - MYSQL_VER: "8.0"
-    - <<: *main
-      env:
-      - PY_VER: "3.7"
-      - MYSQL_VER: "8.0"
-    - <<: *main
-      env:
-      - PY_VER: "3.6"
-      - MYSQL_VER: "8.0"
-    - <<: *main
-      env:
-      - PY_VER: "3.5"
-      - MYSQL_VER: "8.0"
     - <<: *main
       env:
       - PY_VER: "3.8"
       - MYSQL_VER: "5.6"
-    - <<: *main
-      env:
-      - PY_VER: "3.7"
-      - MYSQL_VER: "5.6"
-    - <<: *main
-      env:
-      - PY_VER: "3.6"
-      - MYSQL_VER: "5.6"
-    - <<: *main
-      env:
-      - PY_VER: "3.5"
-      - MYSQL_VER: "5.6"
+    - stage: "Lint: Style"
+      language: python
+      install:
+        - pip install flake8
+      script:
+        - |
+          flake8 --ignore=E121,E123,E126,E226,E24,E704,W503,W504,E722,F401,W605 datajoint \
+          --count --max-complexity=62 --max-line-length=127 --statistics

CHANGELOG.md

@@ -1,5 +1,15 @@
 ## Release notes
 
+### 0.12.6b1 -- May 18, 2020
+* Add `order_by` to `dj.kill` (#668, #779) PR #775, #783
+* Add explicit S3 bucket and file storage location existence checks (#748) PR #781
+* Modify `_update` to allow nullable updates for strings/date (#664) PR #760
+* Avoid logging events on auxiliary tables (#737) PR #753
+* Add `kill_quick` and expand display to include host (#740) PR #741
+* Bugfix - pandas insert fails due to additional `index` field (#666) PR #776
+* Bugfix - `delete_external_files=True` does not remove from S3 (#686) PR #781
+* Bugfix - pandas fetch throws error when `fetch_format='frame'` PR #774
+
 ### 0.12.5b1 -- Feb 27, 2020
 * Support DataJoint datatype and connection plugins (#715, #729) PR 730, #735
 

Dockerfile

@@ -1,5 +1,5 @@
 FROM datajoint/pydev
 
-ADD . /src
-RUN pip install /src && \
-    rm -rf /src
+COPY --chown=dja . /tmp/src
+RUN pip install --user /tmp/src && \
+    rm -rf /tmp/src

LNX-docker-compose.yml

@@ -3,14 +3,55 @@ x-net: &net
   networks:
       - main
 services:
-  app:
+  db:
     <<: *net
-    image: datajoint/pydev:${PY_VER}-alpine${ALPINE_VER}
+    image: datajoint/mysql:$MYSQL_VER
+    environment:
+      - MYSQL_ROOT_PASSWORD=simple
+    # ports:
+    #   - "3306:3306"
+    # volumes:
+    #   - ./mysql/data:/var/lib/mysql
+  minio:
+    <<: *net
+    image: minio/minio:$MINIO_VER
+    environment:
+      - MINIO_ACCESS_KEY=datajoint
+      - MINIO_SECRET_KEY=datajoint
+    # ports:
+    #   - "9000:9000"
+    # volumes:
+    #   - ./minio/config:/root/.minio
+    #   - ./minio/data:/data
+    command: server --address ":9000" /data
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"]
+      timeout: 5s
+      retries: 60
+      interval: 1s
+  fakeservices.datajoint.io:
+    <<: *net
+    image: raphaelguzman/nginx:v0.0.6
+    environment:
+    - ADD_db_TYPE=DATABASE
+    - ADD_db_ENDPOINT=db:3306
+    - ADD_minio_TYPE=MINIO
+    - ADD_minio_ENDPOINT=minio:9000
+    - ADD_minio_PORT=80 # allow unencrypted connections
+    - ADD_minio_PREFIX=/datajoint
+    # ports:
+    #   - "80:80"
+    #   - "443:443"
+    #   - "3306:3306"
     depends_on:
       db:
         condition: service_healthy
       minio:
         condition: service_healthy
+  app:
+    <<: *net
+    image: datajoint/pydev:${PY_VER}-alpine${ALPINE_VER}
+    depends_on:
       fakeservices.datajoint.io:
         condition: service_healthy
     environment:
@@ -20,10 +61,10 @@ services:
       - DJ_TEST_HOST=fakeservices.datajoint.io
       - DJ_TEST_USER=datajoint
       - DJ_TEST_PASSWORD=datajoint
-      - S3_ENDPOINT=fakeservices.datajoint.io:9000
+      - S3_ENDPOINT=fakeservices.datajoint.io
       - S3_ACCESS_KEY=datajoint
       - S3_SECRET_KEY=datajoint
-      - S3_BUCKET=datajoint-test
+      - S3_BUCKET=datajoint.test
       - PYTHON_USER=dja
       - JUPYTER_PASSWORD=datajoint
       - DISPLAY
@@ -35,8 +76,8 @@ services:
        "
         pip install --user nose nose-cov coveralls .;
         pip freeze | grep datajoint;
-        coveralls;
-        nosetests -vsw tests --with-coverage --cover-package=datajoint;
+        nosetests -vsw tests --with-coverage --cover-package=datajoint && coveralls;
+        # jupyter notebook;
        "
     # ports:
     #   - "8888:8888"
@@ -45,55 +86,5 @@ services:
       - .:/src
       - /tmp/.X11-unix:/tmp/.X11-unix:rw
       # - ./notebooks:/home/dja/notebooks
-  db:
-    <<: *net
-    image: datajoint/mysql:$MYSQL_VER
-    environment:
-      - MYSQL_ROOT_PASSWORD=simple
-    # ports:
-    #   - "3306:3306"
-    # volumes:
-    #   - ./mysql/data:/var/lib/mysql
-  minio:
-    <<: *net
-    image: minio/minio:$MINIO_VER
-    environment:
-      - MINIO_ACCESS_KEY=datajoint
-      - MINIO_SECRET_KEY=datajoint
-    # ports:
-    #   - "9000:9000"
-    # volumes:
-    #   - ./minio/config:/root/.minio
-    #   - ./minio/data:/data
-    command: server /data
-    healthcheck:
-      test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"]
-      timeout: 5s
-      retries: 60
-      interval: 1s
-  fakeservices.datajoint.io:
-    <<: *net
-    image: nginx:alpine
-    environment:
-      - URL=datajoint.io
-      - SUBDOMAINS=fakeservices
-      - MINIO_SERVER=http://minio:9000
-      - MYSQL_SERVER=db:3306
-    entrypoint: /entrypoint.sh
-    healthcheck:
-      test: wget --quiet --tries=1 --spider https://fakeservices.datajoint.io:443/minio/health/live || exit 1
-      timeout: 5s
-      retries: 300
-      interval: 1s
-    # ports:
-    #   - "9000:9000"
-    #   - "443:443"
-    #   - "3306:3306"
-    volumes:
-      - ./tests/nginx/base.conf:/base.conf
-      - ./tests/nginx/nginx.conf:/nginx.conf
-      - ./tests/nginx/entrypoint.sh:/entrypoint.sh
-      - ./tests/nginx/fullchain.pem:/certs/fullchain.pem
-      - ./tests/nginx/privkey.pem:/certs/privkey.pem
 networks:
   main:

datajoint/admin.py

@@ -21,33 +21,38 @@ def set_password(new_password=None, connection=None, update_config=None):   # pr
         config.save_local(verbose=True)
 
 
-def kill(restriction=None, connection=None):  # pragma: no cover
+def kill(restriction=None, connection=None, order_by=None):  # pragma: no cover
     """
     view and kill database connections.
     :param restriction: restriction to be applied to processlist
     :param connection: a datajoint.Connection object. Default calls datajoint.conn()
+    :param order_by: order by a single attribute or the list of attributes. defaults to 'id'.
 
     Restrictions are specified as strings and can involve any of the attributes of
     information_schema.processlist: ID, USER, HOST, DB, COMMAND, TIME, STATE, INFO.
 
     Examples:
         dj.kill('HOST LIKE "%compute%"') lists only connections from hosts containing "compute".
-        dj.kill('TIME > 600') lists only connections older than 10 minutes.
+        dj.kill('TIME > 600') lists only connections in their current state for more than 10 minutes
     """
 
     if connection is None:
         connection = conn()
 
+    if order_by is not None and not isinstance(order_by, str):
+        order_by = ','.join(order_by)
+
     query = 'SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()' + (
-        "" if restriction is None else ' AND (%s)' % restriction)
+        "" if restriction is None else ' AND (%s)' % restriction) + (
+            ' ORDER BY %s' % (order_by or 'id'))
 
     while True:
-        print('  ID USER         STATE         TIME  INFO')
-        print('+--+ +----------+ +-----------+ +--+')
+        print('  ID USER         HOST          STATE         TIME    INFO')
+        print('+--+ +----------+ +-----------+ +-----------+ +-----+')
         cur = connection.query(query, as_dict=True)
         for process in cur:
             try:
-                print('{ID:>4d} {USER:<12s} {STATE:<12s} {TIME:>5d}  {INFO}'.format(**process))
+                print('{ID:>4d} {USER:<12s} {HOST:<12s} {STATE:<12s} {TIME:>7d}  {INFO}'.format(**process))
             except TypeError:
                 print(process)
         response = input('process to kill or "q" to quit > ')
@@ -63,3 +68,29 @@ def kill(restriction=None, connection=None):  # pragma: no cover
                     connection.query('kill %d' % pid)
                 except pymysql.err.InternalError:
                     print('Process not found')
+
+
+def kill_quick(restriction=None, connection=None):
+    """
+    Kill database connections without prompting. Returns number of terminated connections.
+    :param restriction: restriction to be applied to processlist
+    :param connection: a datajoint.Connection object. Default calls datajoint.conn()
+
+    Restrictions are specified as strings and can involve any of the attributes of
+    information_schema.processlist: ID, USER, HOST, DB, COMMAND, TIME, STATE, INFO.
+
+    Examples:
+        dj.kill('HOST LIKE "%compute%"') terminates connections from hosts containing "compute".
+    """
+    if connection is None:
+        connection = conn()
+
+    query = 'SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()' + (
+        "" if restriction is None else ' AND (%s)' % restriction)
+
+    cur = connection.query(query, as_dict=True)
+    nkill = 0
+    for process in cur:
+        connection.query('kill %d' % process['ID'])
+        nkill += 1
+    return nkill

datajoint/attribute_adapter.py

@@ -40,7 +40,7 @@ def get_adapter(context, adapter_name):
     adapter_name = adapter_name.lstrip('<').rstrip('>')
     try:
         adapter = (context[adapter_name] if adapter_name in context
-                    else type_plugins[adapter_name]['object'].load())
+                   else type_plugins[adapter_name]['object'].load())
     except KeyError:
         raise DataJointError(
             "Attribute adapter '{adapter_name}' is not defined.".format(adapter_name=adapter_name))

datajoint/autopopulate.py

@@ -5,7 +5,7 @@
 import random
 import inspect
 from tqdm import tqdm
-from .expression import QueryExpression, AndList, U
+from .expression import QueryExpression, AndList
 from .errors import DataJointError, LostConnectionError
 from .table import FreeTable
 import signal

datajoint/blob.py

@@ -73,10 +73,11 @@ def __init__(self, squeeze=False):
 
     def set_dj0(self):
         if not config.get('enable_python_native_blobs'):
-            raise DataJointError('v0.12+ python native blobs disabled. see also: https://github.com/datajoint/datajoint-python#python-native-blobs')
+            raise DataJointError("""v0.12+ python native blobs disabled.
+                See also: https://github.com/datajoint/datajoint-python#python-native-blobs""")
 
         self.protocol = b"dj0\0"  # when using new blob features
-            
+
     def squeeze(self, array, convert_to_scalar=True):
         """
         Simplify the input array - squeeze out all singleton dimensions.
@@ -308,7 +309,7 @@ def read_string(self):
     def pack_string(s):
         blob = s.encode()
         return b"\5" + len_u64(blob) + blob
-    
+
     def read_bytes(self):
         return self.read_binary(self.read_value())
 
@@ -346,7 +347,7 @@ def pack_set(self, t):
 
     def read_dict(self):
         return OrderedDict((self.read_blob(self.read_value()), self.read_blob(self.read_value()))
-                    for _ in range(self.read_value()))
+                           for _ in range(self.read_value()))
 
     def pack_dict(self, d):
         return b"\4" + len_u64(d) + b"".join(
@@ -428,7 +429,7 @@ def read_zero_terminated_string(self):
         data = self._blob[self._pos:target].decode()
         self._pos = target + 1
         return data
-        
+
     def read_value(self, dtype='uint64', count=1):
         data = np.frombuffer(self._blob, dtype=dtype, count=count, offset=self._pos)
         self._pos += data.dtype.itemsize * data.size