Merge branch 'master' into populate_success_count

Author: Thinh Nguyen

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve
 title: ''
-labels: 'bug, awaiting-triage'
+labels: 'bug'
 assignees: ''
 
 ---

.github/workflows/development.yaml

@@ -3,31 +3,55 @@ on:
   push:
     branches:
       - '**' # every branch
+      - '!gh-pages' # exclude gh-pages branch
       - '!stage*' # exclude branches beginning with stage
+    tags:
+      - '\d+\.\d+\.\d+' # only semver tags
   pull_request:
     branches:
       - '**' # every branch
+      - '!gh-pages' # exclude gh-pages branch
       - '!stage*' # exclude branches beginning with stage
 jobs:
-  build-docs:
+  build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - py_ver: 3.8
+            distro: alpine
+            image: djbase
     env:
+      PY_VER: ${{matrix.py_ver}}
+      DISTRO: ${{matrix.distro}}
+      IMAGE: ${{matrix.image}}
       DOCKER_CLIENT_TIMEOUT: "120"
       COMPOSE_HTTP_TIMEOUT: "120"
     steps:
       - uses: actions/checkout@v2
-      - name: Compile docs static artifacts
+      - name: Validate version and release notes
+        run: |
+          DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py)
+          RELEASE_BODY=$(python -c \
+            'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \
+          )
+          echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
+          echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV
+          echo "$RELEASE_BODY" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - name: Build pip artifacts
         run: |
           export HOST_UID=$(id -u)
-          docker-compose -f ./docs-api/docker-compose.yaml up --exit-code-from docs-builder --build 
-      - name: Add docs static artifacts
+          docker-compose -f docker-compose-build.yaml up --exit-code-from app --build
+          echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
+      - if: matrix.py_ver == '3.8' && matrix.distro == 'alpine'
+        name: Add pip artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: docs-api-static
-          path: docs-api/build/html
+          name: pip-datajoint-${{env.DJ_VERSION}}
+          path: dist
           retention-days: 1
   test:
-    if: github.event_name == 'push' || github.event_name == 'pull_request'
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -39,7 +63,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{matrix.py_ver}}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{matrix.py_ver}}
       - name: Install dependencies
@@ -50,58 +74,121 @@ jobs:
         run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
       - name: Run primary tests
         env:
-          UID: "1001"
-          GID: "121"
           PY_VER: ${{matrix.py_ver}}
           MYSQL_VER: ${{matrix.mysql_ver}}
           ALPINE_VER: "3.10"
           MINIO_VER: RELEASE.2021-09-03T03-56-13Z
           COMPOSE_HTTP_TIMEOUT: "120"
           COVERALLS_SERVICE_NAME: travis-ci
           COVERALLS_REPO_TOKEN: fd0BoXG46TPReEem0uMy7BJO5j0w1MQiY
-        run: docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
+        run: |
+          export HOST_UID=$(id -u)
+          export HOST_GID=$(id -g)
+          docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
       - name: Run style tests
         run: |
-          flake8 --ignore=E203,E722,F401,W503 datajoint \
+          flake8 --ignore=E203,E722,W503 datajoint \
                  --count --max-complexity=62 --max-line-length=127 --statistics
           black datajoint --check -v
           black tests --check -v
   publish-docs:
     if: |
       github.event_name == 'push' &&
-      (
-        github.repository_owner == 'datajoint' ||
-        github.repository_owner == 'datajoint-company' ||
-        github.repository_owner == 'dj-sciops'
-      )
-    needs: build-docs
+      startsWith(github.ref, 'refs/tags')
+    needs: test
     runs-on: ubuntu-latest
+    env:
+      DOCKER_CLIENT_TIMEOUT: "120"
+      COMPOSE_HTTP_TIMEOUT: "120"
     steps:
       - uses: actions/checkout@v2
-      - name: Fetch docs static artifacts
+      - name: Deploy docs
+        run: |
+          export MODE=BUILD
+          export PACKAGE=datajoint
+          export UPSTREAM_REPO=https://github.com/${GITHUB_REPOSITORY}.git
+          export HOST_UID=$(id -u)
+          docker compose -f docs/docker-compose.yaml up --exit-code-from docs --build
+          git push origin gh-pages
+  publish-release:
+    if: |
+      github.event_name == 'push' &&
+      startsWith(github.ref, 'refs/tags')
+    needs: test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - py_ver: 3.8
+            distro: alpine
+            image: djbase
+    env:
+      PY_VER: ${{matrix.py_ver}}
+      DISTRO: ${{matrix.distro}}
+      IMAGE: ${{matrix.image}}
+      TWINE_USERNAME: ${{secrets.twine_username}}
+      TWINE_PASSWORD: ${{secrets.twine_password}}
+      DOCKER_CLIENT_TIMEOUT: "120"
+      COMPOSE_HTTP_TIMEOUT: "120"
+    outputs:
+      release_upload_url: ${{steps.create_gh_release.outputs.upload_url}}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{matrix.py_ver}}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{matrix.py_ver}}
+      - name: Determine package version
+        run: |
+          DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py)
+          RELEASE_BODY=$(python -c \
+            'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \
+          )
+          echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
+          echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV
+          echo "$RELEASE_BODY" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - name: Create GH release
+        id: create_gh_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        with:
+          tag_name: ${{env.DJ_VERSION}}
+          release_name: Release ${{env.DJ_VERSION}}
+          body: ${{env.RELEASE_BODY}}
+          prerelease: false
+          draft: false
+      - name: Fetch pip artifacts
         uses: actions/download-artifact@v2
         with:
-          name: docs-api-static
-          path: docs-api/build/html
-      - name: Commit documentation changes
+          name: pip-datajoint-${{env.DJ_VERSION}}
+          path: dist
+      - name: Determine pip artifact paths
         run: |
-          git clone https://github.com/${GITHUB_REPOSITORY}.git \
-              --branch gh-pages --single-branch gh-pages
-          rm -R gh-pages/*
-          cp -r docs-api/build/html/* gh-pages/
-          cp .gitignore gh-pages/
-          touch gh-pages/.nojekyll
-          echo "docs-api.datajoint.org" > gh-pages/CNAME
-          cd gh-pages
-          git config --local user.email "[email protected]"
-          git config --local user.name "GitHub Action"
-          git add . --all
-          git commit -m "Update documentation" -a || true
-          # The above command will fail if no changes were present, so we ignore
-          # the return code.
-      - name: Push changes
-        uses: ad-m/github-push-action@master
+          echo "DJ_WHEEL_PATH=$(ls dist/datajoint-*.whl)" >> $GITHUB_ENV
+          echo "DJ_SDIST_PATH=$(ls dist/datajoint-*.tar.gz)" >> $GITHUB_ENV
+      - name: Upload pip wheel asset to release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
         with:
-          branch: gh-pages
-          directory: gh-pages
-          github_token: ${{secrets.GITHUB_TOKEN}}
+          upload_url: ${{steps.create_gh_release.outputs.upload_url}}
+          asset_path: ${{env.DJ_WHEEL_PATH}}
+          asset_name: pip-datajoint-${{env.DJ_VERSION}}.whl
+          asset_content_type: application/zip
+      - name: Upload pip sdist asset to release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        with:
+          upload_url: ${{steps.create_gh_release.outputs.upload_url}}
+          asset_path: ${{env.DJ_SDIST_PATH}}
+          asset_name: pip-datajoint-${{env.DJ_VERSION}}.tar.gz
+          asset_content_type: application/gzip
+      - name: Publish pip release
+        run: |
+          export HOST_UID=$(id -u)
+          docker-compose -f docker-compose-build.yaml run \
+            -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} app \
+            sh -lc "pip install twine && python -m twine upload dist/*"

.gitignore

@@ -24,6 +24,6 @@ notebook
 .vscode
 __main__.py
 jupyter_custom.js
-apk_requirements.txt
 .eggs
 *.code-workspace
+docs/site

CHANGELOG.md

@@ -1,8 +1,15 @@
 ## Release notes
 
+### 0.14.0 -- TBA
+* Bugfix - Activating a schema requires all tables to exist even if `create_tables=False` PR [#1058](https://github.com/datajoint/datajoint-python/pull/1058)
+
+### 0.13.8 -- Sep 21, 2022
+* Add - New documentation structure based on markdown PR [#1052](https://github.com/datajoint/datajoint-python/pull/1052)
+* Bugfix - Fix queries with backslashes ([#999](https://github.com/datajoint/datajoint-python/issues/999)) PR [#1052](https://github.com/datajoint/datajoint-python/pull/1052)
+
 ### 0.13.7 -- Jul 13, 2022
-* Bugfix - Fix networkx incompatable change by version pinning to 2.6.3 PR #1036 (#1035)
-* Add - Support for serializing numpy datetime64 types PR #1036 (#1022)
+* Bugfix - Fix networkx incompatable change by version pinning to 2.6.3 (#1035) PR #1036
+* Add - Support for serializing numpy datetime64 types (#1022) PR #1036
 * Update - Add traceback to default logging PR #1036
 
 ### 0.13.6 -- Jun 13, 2022
@@ -266,7 +273,7 @@ Documentation and tutorials available at https://docs.datajoint.io and https://t
 
 ### 0.3.4
 * Added method the `ERD.add_parts` method, which adds the part tables of all tables currently in the ERD.
-* `ERD() + arg` and `ERD() - arg` can now accept relation classes as arg.
+* `ERD() + arg` and `ERD() - arg` can now accept table classes as arg.
 
 ### 0.3.3
 * Suppressed warnings (redirected them to logging).  Previoiusly, scipy would throw warnings in ERD, for example.
@@ -276,5 +283,5 @@ Documentation and tutorials available at https://docs.datajoint.io and https://t
 
 ### 0.3.2.
 * Fixed issue #223:  `insert` can insert relations without fetching.
-* ERD() now takes the `context` argument, which specifies in which context to look for classes. The default is taken from the argument (schema or relation).
+* ERD() now takes the `context` argument, which specifies in which context to look for classes. The default is taken from the argument (schema or table).
 * ERD.draw() no longer has the `prefix` argument: class names are shown as found in the context.

Dockerfile

@@ -1,5 +1,13 @@
-FROM datajoint/pydev
-
-COPY --chown=dja . /tmp/src
-RUN pip install --user /tmp/src && \
-    rm -rf /tmp/src
+ARG PY_VER=3.8
+ARG DISTRO=alpine
+ARG IMAGE=djbase
+FROM datajoint/${IMAGE}:py${PY_VER}-${DISTRO}
+WORKDIR /main
+COPY --chown=anaconda:anaconda ./requirements.txt ./setup.py \
+    /main/
+COPY --chown=anaconda:anaconda ./datajoint/*.py /main/datajoint/
+RUN \
+    umask u+rwx,g+rwx,o-rwx && \
+    pip install --no-cache-dir . && \
+    rm -R ./*
+CMD ["python"]

LNX-docker-compose.yml

@@ -1,4 +1,4 @@
-# docker-compose -f LNX-docker-compose.yml --env-file LNX.env up --exit-code-from app --build
+# docker compose -f LNX-docker-compose.yml --env-file LNX.env up --exit-code-from app --build
 version: '2.2'
 x-net: &net
   networks:
@@ -32,7 +32,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: datajoint/nginx:v0.2.1
+    image: datajoint/nginx:v0.2.3
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306
@@ -85,7 +85,7 @@ services:
         # jupyter notebook
     # ports:
     #   - "8888:8888"
-    user: ${UID}:${GID}
+    user: ${HOST_UID}:${HOST_GID}
     volumes:
       - .:/src
       - /tmp/.X11-unix:/tmp/.X11-unix:rw

Makefile

@@ -132,8 +132,8 @@ PY_VER=3.7
 ALPINE_VER=3.10
 MYSQL_VER=5.7
 MINIO_VER=RELEASE.2021-09-03T03-56-13Z
-UID=1000
-GID=1000
+HOST_UID=1000
+HOST_GID=1000
 ```
 * `cp local-docker-compose.yml docker-compose.yml`
 * `docker-compose up -d` (Note configured `JUPYTER_PASSWORD`)

README.md

@@ -50,6 +50,8 @@
     "DataJointError",
     "key",
     "key_hash",
+    "logger",
+    "migrate_dj011_external_blob_storage_to_dj012",
 ]
 
 from .logging import logger

datajoint/__init__.py

@@ -43,8 +43,8 @@ def _call_populate1(key):
 
 class AutoPopulate:
     """
-    AutoPopulate is a mixin class that adds the method populate() to a Relation class.
-    Auto-populated relations must inherit from both Relation and AutoPopulate,
+    AutoPopulate is a mixin class that adds the method populate() to a Table class.
+    Auto-populated tables must inherit from both Table and AutoPopulate,
     must define the property `key_source`, and must define the callback method `make`.
     """
 
@@ -117,7 +117,7 @@ def _job_key(self, key):
 
     def _jobs_to_do(self, restrictions):
         """
-        :return: the relation containing the keys to be computed (derived from self.key_source)
+        :return: the query yeilding the keys to be computed (derived from self.key_source)
         """
         if self.restriction:
             raise DataJointError(