Merge branch 'master' of https://github.com/datajoint/datajoint-python into key_populate

Author: dimitri-yatsenko

.github/workflows/development.yaml

@@ -9,6 +9,23 @@ on:
       - '**' # every branch
       - '!stage*' # exclude branches beginning with stage
 jobs:
+  build-docs:
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_CLIENT_TIMEOUT: "120"
+      COMPOSE_HTTP_TIMEOUT: "120"
+    steps:
+      - uses: actions/checkout@v2
+      - name: Compile docs static artifacts
+        run: |
+          export HOST_UID=$(id -u)
+          docker-compose -f ./docs-api/docker-compose.yaml up --exit-code-from docs-builder --build 
+      - name: Add docs static artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: docs-api-static
+          path: docs-api/build/html
+          retention-days: 1
   test:
     if: github.event_name == 'push' || github.event_name == 'pull_request'
     runs-on: ubuntu-latest
@@ -30,7 +47,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install flake8
+          pip install flake8 black
       - name: Run syntax tests
         run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
       - name: Run primary tests
@@ -47,5 +64,46 @@ jobs:
         run: docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
       - name: Run style tests
         run: |
-          flake8 --ignore=E121,E123,E126,E226,E24,E704,W503,W504,E722,F401,W605 datajoint \
+          flake8 --ignore=E203,E722,F401,W503 datajoint \
                  --count --max-complexity=62 --max-line-length=127 --statistics
+          black datajoint --check -v
+          black tests --check -v
+  publish-docs:
+    if: |
+      github.event_name == 'push' &&
+      (
+        github.repository_owner == 'datajoint' ||
+        github.repository_owner == 'datajoint-company' ||
+        github.repository_owner == 'dj-sciops'
+      )
+    needs: build-docs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Fetch docs static artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: docs-api-static
+          path: docs-api/build/html
+      - name: Commit documentation changes
+        run: |
+          git clone https://github.com/${GITHUB_REPOSITORY}.git \
+              --branch gh-pages --single-branch gh-pages
+          rm -R gh-pages/*
+          cp -r docs-api/build/html/* gh-pages/
+          cp .gitignore gh-pages/
+          touch gh-pages/.nojekyll
+          echo "docs-api.datajoint.org" > gh-pages/CNAME
+          cd gh-pages
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git add . --all
+          git commit -m "Update documentation" -a || true
+          # The above command will fail if no changes were present, so we ignore
+          # the return code.
+      - name: Push changes
+        uses: ad-m/github-push-action@master
+        with:
+          branch: gh-pages
+          directory: gh-pages
+          github_token: ${{secrets.GITHUB_TOKEN}}

CHANGELOG.md

@@ -1,5 +1,16 @@
 ## Release notes
 
+### 0.13.5 -- May 19, 2022
+* Update - Import ABC from collections.abc for Python 3.10 compatibility
+* Bugfix - Fix multiprocessing value error (#1013) PR #1026
+
+### 0.13.4 -- March, 28 2022
+* Add - Allow reading blobs produced by legacy 32-bit compiled mYm library for matlab. PR #995
+* Bugfix - Add missing `jobs` argument for multiprocessing PR #997
+* Add - Test for multiprocessing PR #1008
+* Bugfix - Fix external store key name doesn't allow '-' (#1005) PR #1006
+* Add - Adopted black formatting into code base PR #998
+
 ### 0.13.3 -- Feb 9, 2022
 * Bugfix - Fix error in listing ancestors, descendants with part tables.
 * Bugfix - Fix Python 3.10 compatibility (#983) PR #972

LNX-docker-compose.yml

@@ -32,7 +32,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: datajoint/nginx:v0.0.19
+    image: datajoint/nginx:v0.1.1
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306

README.md

@@ -8,10 +8,8 @@
 # Welcome to DataJoint for Python!
 DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
 
-DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers.
-
-Vathes LLC supports DataJoint for Python as an open-source project and everyone is welcome to contribute.
-Its DataJoint Neuro (https://djneuro.io) business provides support to neuroscience labs for developing and executing custom data pipelines.
+DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab at Baylor College of Medicine for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers.
+Presently, the primary developer of DataJoint open-source software is the company DataJoint (https://datajoint.com). Related resources are listed at https://datajoint.org.
 
 ## Installation
 ```
@@ -22,7 +20,25 @@ If you already have an older version of DataJoint installed using `pip`, upgrade
 ```bash
 pip3 install --upgrade datajoint
 ```
+
+## Documentation and Tutorials
+
+* https://datajoint.org  -- start page
+* https://docs.datajoint.org -- up-to-date documentation
+* https://tutorials.datajoint.io -- step-by-step tutorials
+* https://elements.datajoint.org -- catalog of example pipelines
+* https://codebook.datajoint.io -- interactive online tutorials
+
+## Citation
++ If your work uses DataJoint for Python, please cite the following Research Resource Identifier (RRID) and manuscript.
+
++ DataJoint ([RRID:SCR_014543](https://scicrunch.org/resolver/SCR_014543)) - DataJoint for Python (version `<Enter version number>`)
+
++ Yatsenko D, Reimer J, Ecker AS, Walker EY, Sinz F, Berens P, Hoenselaar A, Cotton RJ, Siapas AS, Tolias AS. DataJoint: managing big scientific data using MATLAB or Python. bioRxiv. 2015 Jan 1:031658. doi: https://doi.org/10.1101/031658
+
 ## Python Native Blobs
+<details>
+<summary>Click to expand details</summary>
 
 DataJoint 0.12 adds full support for all native python data types in blobs: tuples, lists, sets, dicts, strings, bytes, `None`, and all their recursive combinations.
 The new blobs are a superset of the old functionality and are fully backward compatible.
@@ -92,16 +108,23 @@ the situation, but generally the following strategies may apply:
 As always, be sure that your data is safely backed up before modifying any
 important DataJoint schema or records.
 
-## Documentation and Tutorials
-A number of labs are currently adopting DataJoint and we are quickly getting the documentation in shape in February 2017.
+</details>
 
-* https://datajoint.io  -- start page
-* https://docs.datajoint.io -- up-to-date documentation
-* https://tutorials.datajoint.io -- step-by-step tutorials
-* https://catalog.datajoint.io -- catalog of example pipelines
+### API docs
 
-## Running Tests Locally
+The API documentation can be built using sphinx by running
 
+``` bash
+pip install sphinx sphinx_rtd_theme 
+(cd docs-api/sphinx && make html)
+```
+
+Generated docs are written to `docs-api/docs/html/index.html`.
+More details in [docs-api/README.md](docs-api/README.md).
+
+## Running Tests Locally
+<details>
+<summary>Click to expand details</summary>
 
 * Create an `.env` with desired development environment values e.g.
 ``` sh
@@ -136,9 +159,9 @@ GID=1000
 * Add entry in `/etc/hosts` for `127.0.0.1 fakeservices.datajoint.io`
 
 
-
-
 ### Launch Jupyter Notebook for Interactive Use
 * Navigate to `localhost:8888`
 * Input Jupyter password
 * Launch a notebook i.e. `New > Python 3`
+
+</details>

datajoint/__init__.py

@@ -10,21 +10,47 @@
 that any use of DataJoint leading to a publication be acknowledged in the publication.
 
 Please cite:
-    http://biorxiv.org/content/early/2015/11/14/031658
-    http://dx.doi.org/10.1101/031658
+
+  - http://biorxiv.org/content/early/2015/11/14/031658
+  - http://dx.doi.org/10.1101/031658
 """
 
 __author__ = "DataJoint Contributors"
 __date__ = "November 7, 2020"
-__all__ = ['__author__', '__version__',
-           'config', 'conn', 'Connection',
-           'Schema', 'schema', 'VirtualModule', 'create_virtual_module',
-           'list_schemas', 'Table', 'FreeTable',
-           'Manual', 'Lookup', 'Imported', 'Computed', 'Part',
-           'Not', 'AndList', 'U', 'Diagram', 'Di', 'ERD',
-           'set_password', 'kill',
-           'MatCell', 'MatStruct', 'AttributeAdapter',
-           'errors', 'DataJointError', 'key', 'key_hash']
+__all__ = [
+    "__author__",
+    "__version__",
+    "config",
+    "conn",
+    "Connection",
+    "Schema",
+    "schema",
+    "VirtualModule",
+    "create_virtual_module",
+    "list_schemas",
+    "Table",
+    "FreeTable",
+    "Manual",
+    "Lookup",
+    "Imported",
+    "Computed",
+    "Part",
+    "Not",
+    "AndList",
+    "U",
+    "Diagram",
+    "Di",
+    "ERD",
+    "set_password",
+    "kill",
+    "MatCell",
+    "MatStruct",
+    "AttributeAdapter",
+    "errors",
+    "DataJointError",
+    "key",
+    "key_hash",
+]
 
 from .version import __version__
 from .settings import config
@@ -44,6 +70,6 @@
 from .errors import DataJointError
 from .migrate import migrate_dj011_external_blob_storage_to_dj012
 
-ERD = Di = Diagram                      # Aliases for Diagram
-schema = Schema                         # Aliases for Schema
-create_virtual_module = VirtualModule   # Aliases for VirtualModule
+ERD = Di = Diagram  # Aliases for Diagram
+schema = Schema  # Aliases for Schema
+create_virtual_module = VirtualModule  # Aliases for VirtualModule

datajoint/admin.py

@@ -5,25 +5,30 @@
 from .utils import user_choice
 
 
-def set_password(new_password=None, connection=None, update_config=None):   # pragma: no cover
+def set_password(
+    new_password=None, connection=None, update_config=None
+):  # pragma: no cover
     connection = conn() if connection is None else connection
     if new_password is None:
-        new_password = getpass('New password: ')
-        confirm_password = getpass('Confirm password: ')
+        new_password = getpass("New password: ")
+        confirm_password = getpass("Confirm password: ")
         if new_password != confirm_password:
-            print('Failed to confirm the password! Aborting password change.')
+            print("Failed to confirm the password! Aborting password change.")
             return
     connection.query("SET PASSWORD = PASSWORD('%s')" % new_password)
-    print('Password updated.')
+    print("Password updated.")
 
-    if update_config or (update_config is None and user_choice('Update local setting?') == 'yes'):
-        config['database.password'] = new_password
+    if update_config or (
+        update_config is None and user_choice("Update local setting?") == "yes"
+    ):
+        config["database.password"] = new_password
         config.save_local(verbose=True)
 
 
 def kill(restriction=None, connection=None, order_by=None):  # pragma: no cover
     """
     view and kill database connections.
+
     :param restriction: restriction to be applied to processlist
     :param connection: a datajoint.Connection object. Default calls datajoint.conn()
     :param order_by: order by a single attribute or the list of attributes. defaults to 'id'.
@@ -40,24 +45,32 @@ def kill(restriction=None, connection=None, order_by=None):  # pragma: no cover
         connection = conn()
 
     if order_by is not None and not isinstance(order_by, str):
-        order_by = ','.join(order_by)
+        order_by = ",".join(order_by)
 
-    query = 'SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()' + (
-        "" if restriction is None else ' AND (%s)' % restriction) + (
-            ' ORDER BY %s' % (order_by or 'id'))
+    query = (
+        "SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()"
+        + ("" if restriction is None else " AND (%s)" % restriction)
+        + (" ORDER BY %s" % (order_by or "id"))
+    )
 
     while True:
-        print('  ID USER         HOST          STATE         TIME    INFO')
-        print('+--+ +----------+ +-----------+ +-----------+ +-----+')
-        cur = ({k.lower(): v for k, v in elem.items()}
-               for elem in connection.query(query, as_dict=True))
+        print("  ID USER         HOST          STATE         TIME    INFO")
+        print("+--+ +----------+ +-----------+ +-----------+ +-----+")
+        cur = (
+            {k.lower(): v for k, v in elem.items()}
+            for elem in connection.query(query, as_dict=True)
+        )
         for process in cur:
             try:
-                print('{id:>4d} {user:<12s} {host:<12s} {state:<12s} {time:>7d}  {info}'.format(**process))
+                print(
+                    "{id:>4d} {user:<12s} {host:<12s} {state:<12s} {time:>7d}  {info}".format(
+                        **process
+                    )
+                )
             except TypeError:
                 print(process)
         response = input('process to kill or "q" to quit > ')
-        if response == 'q':
+        if response == "q":
             break
         if response:
             try:
@@ -66,14 +79,15 @@ def kill(restriction=None, connection=None, order_by=None):  # pragma: no cover
                 pass  # ignore non-numeric input
             else:
                 try:
-                    connection.query('kill %d' % pid)
+                    connection.query("kill %d" % pid)
                 except pymysql.err.InternalError:
-                    print('Process not found')
+                    print("Process not found")
 
 
 def kill_quick(restriction=None, connection=None):
     """
     Kill database connections without prompting. Returns number of terminated connections.
+
     :param restriction: restriction to be applied to processlist
     :param connection: a datajoint.Connection object. Default calls datajoint.conn()
 
@@ -86,13 +100,17 @@ def kill_quick(restriction=None, connection=None):
     if connection is None:
         connection = conn()
 
-    query = 'SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()' + (
-        "" if restriction is None else ' AND (%s)' % restriction)
+    query = (
+        "SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()"
+        + ("" if restriction is None else " AND (%s)" % restriction)
+    )
 
-    cur = ({k.lower(): v for k, v in elem.items()}
-           for elem in connection.query(query, as_dict=True))
+    cur = (
+        {k.lower(): v for k, v in elem.items()}
+        for elem in connection.query(query, as_dict=True)
+    )
     nkill = 0
     for process in cur:
-        connection.query('kill %d' % process['id'])
+        connection.query("kill %d" % process["id"])
         nkill += 1
     return nkill