Merge branch 'master' into add-api-docs

Author: guzman-raphael

.github/workflows/development.yaml

@@ -30,7 +30,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install flake8
+          pip install flake8 black
       - name: Run syntax tests
         run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
       - name: Run primary tests
@@ -40,12 +40,14 @@ jobs:
           PY_VER: ${{matrix.py_ver}}
           MYSQL_VER: ${{matrix.mysql_ver}}
           ALPINE_VER: "3.10"
-          MINIO_VER: RELEASE.2019-09-26T19-42-35Z
+          MINIO_VER: RELEASE.2021-09-03T03-56-13Z
           COMPOSE_HTTP_TIMEOUT: "120"
           COVERALLS_SERVICE_NAME: travis-ci
           COVERALLS_REPO_TOKEN: fd0BoXG46TPReEem0uMy7BJO5j0w1MQiY
         run: docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
       - name: Run style tests
         run: |
-          flake8 --ignore=E121,E123,E126,E226,E24,E704,W503,W504,E722,F401,W605 datajoint \
+          flake8 --ignore=E203,E722,F401,W503 datajoint \
                  --count --max-complexity=62 --max-line-length=127 --statistics
+          black datajoint --check -v
+          black tests --check -v

CHANGELOG.md

@@ -1,7 +1,30 @@
 ## Release notes
 
-### 0.13.3 -- May 28, 2021
+### 0.13.4 -- March, 28 2022
+* Add - Allow reading blobs produced by legacy 32-bit compiled mYm library for matlab. PR #995
+* Bugfix - Add missing `jobs` argument for multiprocessing PR #997
+* Add - Test for multiprocessing PR #1008
+* Bugfix - Fix external store key name doesn't allow '-' (#1005) PR #1006
+* Add - Adopted black formatting into code base PR #998
+
+### 0.13.3 -- Feb 9, 2022
+* Bugfix - Fix error in listing ancestors, descendants with part tables.
+* Bugfix - Fix Python 3.10 compatibility (#983) PR #972
+* Bugfix - Allow renaming non-conforming attributes in proj (#982) PR #972
+* Add - Expose proxy feature for S3 external stores (#961) PR #962
+* Add - implement multiprocessing in populate (#695) PR #704, #969
 * Bugfix - Dependencies not properly loaded on populate. (#902) PR #919
+* Bugfix - Replace use of numpy aliases of built-in types with built-in type. (#938) PR #939
+* Bugfix - Deletes and drops must include the master of each part. (#151, #374) PR #957
+* Bugfix - `ExternalTable.delete` should not remove row on error (#953) PR #956
+* Bugfix - Fix error handling of remove_object function in `s3.py` (#952) PR #955
+* Bugfix - Fix regression issue with `DISTINCT` clause and `GROUP_BY` (#914) PR #963
+* Bugfix - Fix sql code generation to comply with sql mode `ONLY_FULL_GROUP_BY` (#916) PR #965
+* Bugfix - Fix count for left-joined `QueryExpressions` (#951) PR #966
+* Bugfix - Fix assertion error when performing a union into a join (#930) PR #967
+* Update `~jobs.error_stack` from blob to mediumblob to allow error stacks >64kB in jobs (#984) PR #986
+* Bugfix - Fix error when performing a union on multiple tables (#926) PR #964
+* Add - Allow optional keyword arguments for `make()` in `populate()` PR #971
 
 ### 0.13.2 -- May 7, 2021
 * Update `setuptools_certificate` dependency to new name `otumat`
@@ -44,13 +67,13 @@
 * Fix display of part tables in `schema.save`. (#821) PR #833
 * Add `schema.list_tables`. (#838) PR #844
 * Fix minio new version regression.  PR #847
-* Add more S3 logging for debugging. (#831) PR #832 
+* Add more S3 logging for debugging. (#831) PR #832
 * Convert testing framework from TravisCI to GitHub Actions (#841) PR #840
- 
+
 ### 0.12.7 -- Oct 27, 2020
 * Fix case sensitivity issues to adapt to MySQL 8+.  PR #819
 * Fix pymysql regression bug (#814) PR #816
-* Adapted attribute types now have dtype=object in all recarray results. PR #811 
+* Adapted attribute types now have dtype=object in all recarray results. PR #811
 
 ### 0.12.6 -- May 15, 2020
 * Add `order_by` to `dj.kill` (#668, #779) PR #775, #783
@@ -122,7 +145,7 @@
 * Fix #628 - incompatibility with pyparsing 2.4.1
 
 ### 0.11.1 -- Nov 15, 2018
-* Fix ordering of attributes in proj (#483 and #516)
+* Fix ordering of attributes in proj (#483, #516)
 * Prohibit direct insert into auto-populated tables (#511)
 
 ### 0.11.0 -- Oct 25, 2018
@@ -142,9 +165,9 @@
 * Bugfix in restriction of the form (A & B) * B (#463)
 * Improved error messages (#466)
 
-### 0.10.0 -- Jan 10, 2018 
+### 0.10.0 -- Jan 10, 2018
 * Deletes are more efficient (#424)
-* ERD shows table definition on tooltip hover in Jupyter (#422) 
+* ERD shows table definition on tooltip hover in Jupyter (#422)
 * S3 external storage
 * Garbage collection for external sorage
 * Most operators and methods of tables can be invoked as class methods rather than instance methods (#407)
@@ -158,7 +181,7 @@
 * Implement union operator +
 * Implement file-based external storage
 
-### 0.8.0 -- Jul 26, 2017 
+### 0.8.0 -- Jul 26, 2017
 Documentation and tutorials available at https://docs.datajoint.io and https://tutorials.datajoint.io
 * improved the ERD graphics and features using the graphviz libraries (#207, #333)
 * improved password handling logic (#322, #321)
@@ -177,11 +200,11 @@ Documentation and tutorials available at https://docs.datajoint.io and https://t
 * Added `dj.create_virtual_module`
 
 ### 0.4.10 (#286) -- Feb 6, 2017
-* Removed Vagrant and Readthedocs support 
+* Removed Vagrant and Readthedocs support
 * Explicit saving of configuration (issue #284)
 
 ### 0.4.9 (#285) -- Feb 2, 2017
-* Fixed setup.py for pip install 
+* Fixed setup.py for pip install
 
 ### 0.4.7 (#281) -- Jan 24, 2017
 * Fixed issues related to order of attributes in projection.
@@ -210,10 +233,10 @@ Documentation and tutorials available at https://docs.datajoint.io and https://t
 
 ### 0.3.8  -- Aug 2, 2016
 * added the `_update` method in `base_relation`. It allows updating values in existing tuples.
-* bugfix in reading values of type double.  Previously it was cast as float32. 
+* bugfix in reading values of type double.  Previously it was cast as float32.
 
 ### 0.3.7  -- Jul 31, 2016
-* added parameter `ignore_extra_fields` in `insert` 
+* added parameter `ignore_extra_fields` in `insert`
 * `insert(..., skip_duplicates=True)` now relies on `SELECT IGNORE`.  Previously it explicitly checked if tuple already exists.
 * table previews now include blob attributes displaying the string <BLOB>
 
@@ -235,7 +258,7 @@ Documentation and tutorials available at https://docs.datajoint.io and https://t
 * ERD() no longer text the context argument.
 * ERD.draw() now takes an optional context argument.  By default uses the caller's locals.
 
-### 0.3.2.   
+### 0.3.2.
 * Fixed issue #223:  `insert` can insert relations without fetching.
 * ERD() now takes the `context` argument, which specifies in which context to look for classes. The default is taken from the argument (schema or relation).
 * ERD.draw() no longer has the `prefix` argument: class names are shown as found in the context.

LNX-docker-compose.yml

@@ -32,7 +32,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: datajoint/nginx:v0.0.16
+    image: datajoint/nginx:v0.1.1
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306

README.md

@@ -8,10 +8,8 @@
 # Welcome to DataJoint for Python!
 DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
 
-DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers.
-
-Vathes LLC supports DataJoint for Python as an open-source project and everyone is welcome to contribute.
-Its DataJoint Neuro (https://djneuro.io) business provides support to neuroscience labs for developing and executing custom data pipelines.
+DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab at Baylor College of Medicine for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers.
+Presently, the primary developer of DataJoint open-source software is the company DataJoint (https://datajoint.com). Related resources are listed at https://datajoint.org.
 
 ## Installation
 ```
@@ -22,7 +20,25 @@ If you already have an older version of DataJoint installed using `pip`, upgrade
 ```bash
 pip3 install --upgrade datajoint
 ```
+
+## Documentation and Tutorials
+
+* https://datajoint.org  -- start page
+* https://docs.datajoint.org -- up-to-date documentation
+* https://tutorials.datajoint.io -- step-by-step tutorials
+* https://elements.datajoint.org -- catalog of example pipelines
+* https://codebook.datajoint.io -- interactive online tutorials
+
+## Citation
++ If your work uses DataJoint for Python, please cite the following Research Resource Identifier (RRID) and manuscript.
+
++ DataJoint ([RRID:SCR_014543](https://scicrunch.org/resolver/SCR_014543)) - DataJoint for Python (version `<Enter version number>`)
+
++ Yatsenko D, Reimer J, Ecker AS, Walker EY, Sinz F, Berens P, Hoenselaar A, Cotton RJ, Siapas AS, Tolias AS. DataJoint: managing big scientific data using MATLAB or Python. bioRxiv. 2015 Jan 1:031658. doi: https://doi.org/10.1101/031658
+
 ## Python Native Blobs
+<details>
+<summary>Click to expand details</summary>
 
 DataJoint 0.12 adds full support for all native python data types in blobs: tuples, lists, sets, dicts, strings, bytes, `None`, and all their recursive combinations.
 The new blobs are a superset of the old functionality and are fully backward compatible.
@@ -92,13 +108,7 @@ the situation, but generally the following strategies may apply:
 As always, be sure that your data is safely backed up before modifying any
 important DataJoint schema or records.
 
-## Documentation and Tutorials
-A number of labs are currently adopting DataJoint and we are quickly getting the documentation in shape in February 2017.
-
-* https://datajoint.io  -- start page
-* https://docs.datajoint.io -- up-to-date documentation
-* https://tutorials.datajoint.io -- step-by-step tutorials
-* https://catalog.datajoint.io -- catalog of example pipelines
+</details>
 
 ### API docs
 
@@ -113,14 +123,15 @@ Generated docs are written to `docs-api/docs/html/index.html`.
 More details in [docs-api/README.md](docs-api/README.md).
 
 ## Running Tests Locally
-
+<details>
+<summary>Click to expand details</summary>
 
 * Create an `.env` with desired development environment values e.g.
 ``` sh
 PY_VER=3.7
 ALPINE_VER=3.10
 MYSQL_VER=5.7
-MINIO_VER=RELEASE.2019-09-26T19-42-35Z
+MINIO_VER=RELEASE.2021-09-03T03-56-13Z
 UID=1000
 GID=1000
 ```
@@ -152,3 +163,5 @@ GID=1000
 * Navigate to `localhost:8888`
 * Input Jupyter password
 * Launch a notebook i.e. `New > Python 3`
+
+</details>

datajoint/__init__.py

@@ -16,15 +16,40 @@
 
 __author__ = "DataJoint Contributors"
 __date__ = "November 7, 2020"
-__all__ = ['__author__', '__version__',
-           'config', 'conn', 'Connection',
-           'Schema', 'schema', 'VirtualModule', 'create_virtual_module',
-           'list_schemas', 'Table', 'FreeTable',
-           'Manual', 'Lookup', 'Imported', 'Computed', 'Part',
-           'Not', 'AndList', 'U', 'Diagram', 'Di', 'ERD',
-           'set_password', 'kill',
-           'MatCell', 'MatStruct', 'AttributeAdapter',
-           'errors', 'DataJointError', 'key', 'key_hash']
+__all__ = [
+    "__author__",
+    "__version__",
+    "config",
+    "conn",
+    "Connection",
+    "Schema",
+    "schema",
+    "VirtualModule",
+    "create_virtual_module",
+    "list_schemas",
+    "Table",
+    "FreeTable",
+    "Manual",
+    "Lookup",
+    "Imported",
+    "Computed",
+    "Part",
+    "Not",
+    "AndList",
+    "U",
+    "Diagram",
+    "Di",
+    "ERD",
+    "set_password",
+    "kill",
+    "MatCell",
+    "MatStruct",
+    "AttributeAdapter",
+    "errors",
+    "DataJointError",
+    "key",
+    "key_hash",
+]
 
 from .version import __version__
 from .settings import config
@@ -44,6 +69,6 @@
 from .errors import DataJointError
 from .migrate import migrate_dj011_external_blob_storage_to_dj012
 
-ERD = Di = Diagram                      # Aliases for Diagram
-schema = Schema                         # Aliases for Schema
-create_virtual_module = VirtualModule   # Aliases for VirtualModule
+ERD = Di = Diagram  # Aliases for Diagram
+schema = Schema  # Aliases for Schema
+create_virtual_module = VirtualModule  # Aliases for VirtualModule

datajoint/admin.py

@@ -5,19 +5,23 @@
 from .utils import user_choice
 
 
-def set_password(new_password=None, connection=None, update_config=None):   # pragma: no cover
+def set_password(
+    new_password=None, connection=None, update_config=None
+):  # pragma: no cover
     connection = conn() if connection is None else connection
     if new_password is None:
-        new_password = getpass('New password: ')
-        confirm_password = getpass('Confirm password: ')
+        new_password = getpass("New password: ")
+        confirm_password = getpass("Confirm password: ")
         if new_password != confirm_password:
-            print('Failed to confirm the password! Aborting password change.')
+            print("Failed to confirm the password! Aborting password change.")
             return
     connection.query("SET PASSWORD = PASSWORD('%s')" % new_password)
-    print('Password updated.')
+    print("Password updated.")
 
-    if update_config or (update_config is None and user_choice('Update local setting?') == 'yes'):
-        config['database.password'] = new_password
+    if update_config or (
+        update_config is None and user_choice("Update local setting?") == "yes"
+    ):
+        config["database.password"] = new_password
         config.save_local(verbose=True)
 
 
@@ -40,24 +44,32 @@ def kill(restriction=None, connection=None, order_by=None):  # pragma: no cover
         connection = conn()
 
     if order_by is not None and not isinstance(order_by, str):
-        order_by = ','.join(order_by)
+        order_by = ",".join(order_by)
 
-    query = 'SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()' + (
-        "" if restriction is None else ' AND (%s)' % restriction) + (
-            ' ORDER BY %s' % (order_by or 'id'))
+    query = (
+        "SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()"
+        + ("" if restriction is None else " AND (%s)" % restriction)
+        + (" ORDER BY %s" % (order_by or "id"))
+    )
 
     while True:
-        print('  ID USER         HOST          STATE         TIME    INFO')
-        print('+--+ +----------+ +-----------+ +-----------+ +-----+')
-        cur = ({k.lower(): v for k, v in elem.items()}
-               for elem in connection.query(query, as_dict=True))
+        print("  ID USER         HOST          STATE         TIME    INFO")
+        print("+--+ +----------+ +-----------+ +-----------+ +-----+")
+        cur = (
+            {k.lower(): v for k, v in elem.items()}
+            for elem in connection.query(query, as_dict=True)
+        )
         for process in cur:
             try:
-                print('{id:>4d} {user:<12s} {host:<12s} {state:<12s} {time:>7d}  {info}'.format(**process))
+                print(
+                    "{id:>4d} {user:<12s} {host:<12s} {state:<12s} {time:>7d}  {info}".format(
+                        **process
+                    )
+                )
             except TypeError:
                 print(process)
         response = input('process to kill or "q" to quit > ')
-        if response == 'q':
+        if response == "q":
             break
         if response:
             try:
@@ -66,9 +78,9 @@ def kill(restriction=None, connection=None, order_by=None):  # pragma: no cover
                 pass  # ignore non-numeric input
             else:
                 try:
-                    connection.query('kill %d' % pid)
+                    connection.query("kill %d" % pid)
                 except pymysql.err.InternalError:
-                    print('Process not found')
+                    print("Process not found")
 
 
 def kill_quick(restriction=None, connection=None):
@@ -86,13 +98,17 @@ def kill_quick(restriction=None, connection=None):
     if connection is None:
         connection = conn()
 
-    query = 'SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()' + (
-        "" if restriction is None else ' AND (%s)' % restriction)
+    query = (
+        "SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()"
+        + ("" if restriction is None else " AND (%s)" % restriction)
+    )
 
-    cur = ({k.lower(): v for k, v in elem.items()}
-           for elem in connection.query(query, as_dict=True))
+    cur = (
+        {k.lower(): v for k, v in elem.items()}
+        for elem in connection.query(query, as_dict=True)
+    )
     nkill = 0
     for process in cur:
-        connection.query('kill %d' % process['id'])
+        connection.query("kill %d" % process["id"])
         nkill += 1
     return nkill