Merge pull request #1 from guzman-raphael/switch-to-djtest

Fix merge conflicts + update images

Author: A-Baji

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve
 title: ''
-labels: 'bug, awaiting-triage'
+labels: 'bug'
 assignees: ''
 
 ---

.github/workflows/development.yaml

@@ -3,48 +3,69 @@ on:
   push:
     branches:
       - '**' # every branch
+      - '!gh-pages' # exclude gh-pages branch
       - '!stage*' # exclude branches beginning with stage
+    tags:
+      - '\d+\.\d+\.\d+' # only semver tags
   pull_request:
     branches:
       - '**' # every branch
+      - '!gh-pages' # exclude gh-pages branch
       - '!stage*' # exclude branches beginning with stage
 jobs:
-  build-docs:
+  build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - py_ver: '3.9'
+            distro: debian
+            image: djbase
     env:
+      PY_VER: ${{matrix.py_ver}}
+      DISTRO: ${{matrix.distro}}
+      IMAGE: ${{matrix.image}}
       DOCKER_CLIENT_TIMEOUT: "120"
       COMPOSE_HTTP_TIMEOUT: "120"
     steps:
-      - uses: actions/checkout@v2
-      - name: Compile docs static artifacts
+      - uses: actions/checkout@v3
+      - name: Validate version and release notes
+        run: |
+          DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py)
+          RELEASE_BODY=$(python -c \
+            'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \
+          )
+          echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
+          echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV
+          echo "$RELEASE_BODY" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - name: Build pip artifacts
         run: |
           export HOST_UID=$(id -u)
-          docker-compose -f ./docs-api/docker-compose.yaml up --exit-code-from docs-builder --build 
-      - name: Add docs static artifacts
-        uses: actions/upload-artifact@v2
+          docker-compose -f docker-compose-build.yaml up --exit-code-from app --build
+          echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
+      - if: matrix.py_ver == '3.9' && matrix.distro == 'debian'
+        name: Add pip artifacts
+        uses: actions/upload-artifact@v3
         with:
-          name: docs-api-static
-          path: docs-api/build/html
+          name: pip-datajoint-${{env.DJ_VERSION}}
+          path: dist
           retention-days: 1
   test:
-    if: github.event_name == 'push' || github.event_name == 'pull_request'
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # py_ver: ["3.10"]
         py_ver: ["3.9"]
         mysql_ver: ["8.0", "5.7"]
         include:
-          # - py_ver: "3.9"
-          #   mysql_ver: "5.7"
           - py_ver: "3.8"
             mysql_ver: "5.7"
           - py_ver: "3.7"
             mysql_ver: "5.7"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{matrix.py_ver}}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{matrix.py_ver}}
       - name: Install dependencies
@@ -55,56 +76,135 @@ jobs:
         run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
       - name: Run primary tests
         env:
-          HOST_UID: "1001"
-          GID: "121"
           PY_VER: ${{matrix.py_ver}}
           MYSQL_VER: ${{matrix.mysql_ver}}
           DISTRO: alpine
           MINIO_VER: RELEASE.2021-09-03T03-56-13Z
+          DOCKER_CLIENT_TIMEOUT: "120"
           COMPOSE_HTTP_TIMEOUT: "120"
-        run: docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
+        run: |
+          export HOST_UID=$(id -u)
+          docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
       - name: Run style tests
         run: |
-          flake8 --ignore=E203,E722,F401,W503 datajoint \
+          flake8 --ignore=E203,E722,W503 datajoint \
                  --count --max-complexity=62 --max-line-length=127 --statistics
           black datajoint --check -v
           black tests --check -v
   publish-docs:
     if: |
       github.event_name == 'push' &&
-      (
-        github.repository_owner == 'datajoint' ||
-        github.repository_owner == 'datajoint-company' ||
-        github.repository_owner == 'dj-sciops'
-      )
-    needs: build-docs
+      startsWith(github.ref, 'refs/tags')
+    needs: test
     runs-on: ubuntu-latest
+    env:
+      DOCKER_CLIENT_TIMEOUT: "120"
+      COMPOSE_HTTP_TIMEOUT: "120"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Deploy docs
+        run: |
+          export MODE=BUILD
+          export PACKAGE=datajoint
+          export UPSTREAM_REPO=https://github.com/${GITHUB_REPOSITORY}.git
+          export HOST_UID=$(id -u)
+          docker compose -f docs/docker-compose.yaml up --exit-code-from docs --build
+          git push origin gh-pages
+  publish-release:
+    if: |
+      github.event_name == 'push' &&
+      startsWith(github.ref, 'refs/tags')
+    needs: test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - py_ver: '3.9'
+            distro: debian
+            image: djbase
+    env:
+      PY_VER: ${{matrix.py_ver}}
+      DISTRO: ${{matrix.distro}}
+      IMAGE: ${{matrix.image}}
+      TWINE_USERNAME: ${{secrets.twine_username}}
+      TWINE_PASSWORD: ${{secrets.twine_password}}
+      DOCKER_CLIENT_TIMEOUT: "120"
+      COMPOSE_HTTP_TIMEOUT: "120"
+    outputs:
+      release_upload_url: ${{steps.create_gh_release.outputs.upload_url}}
     steps:
-      - uses: actions/checkout@v2
-      - name: Fetch docs static artifacts
-        uses: actions/download-artifact@v2
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{matrix.py_ver}}
+        uses: actions/setup-python@v4
         with:
-          name: docs-api-static
-          path: docs-api/build/html
-      - name: Commit documentation changes
+          python-version: ${{matrix.py_ver}}
+      - name: Determine package version
         run: |
-          git clone https://github.com/${GITHUB_REPOSITORY}.git \
-              --branch gh-pages --single-branch gh-pages
-          rm -R gh-pages/*
-          cp -r docs-api/build/html/* gh-pages/
-          cp .gitignore gh-pages/
-          touch gh-pages/.nojekyll
-          echo "docs-api.datajoint.org" > gh-pages/CNAME
-          cd gh-pages
-          git config --local user.email "[email protected]"
-          git config --local user.name "GitHub Action"
-          git add . --all
-          git commit -m "Update documentation" -a || true
-          # The above command will fail if no changes were present, so we ignore
-          # the return code.
-      - name: Push changes
-        uses: ad-m/github-push-action@master
+          DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py)
+          RELEASE_BODY=$(python -c \
+            'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \
+          )
+          echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
+          echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV
+          echo "$RELEASE_BODY" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - name: Create GH release
+        id: create_gh_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
         with:
-          branch: gh-pages
-          directory: gh-pages
-          github_token: ${{secrets.GITHUB_TOKEN}}
+          tag_name: ${{env.DJ_VERSION}}
+          release_name: Release ${{env.DJ_VERSION}}
+          body: ${{env.RELEASE_BODY}}
+          prerelease: false
+          draft: false
+      - name: Fetch pip artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: pip-datajoint-${{env.DJ_VERSION}}
+          path: dist
+      - name: Determine pip artifact paths
+        run: |
+          echo "DJ_WHEEL_PATH=$(ls dist/datajoint-*.whl)" >> $GITHUB_ENV
+          echo "DJ_SDIST_PATH=$(ls dist/datajoint-*.tar.gz)" >> $GITHUB_ENV
+      - name: Upload pip wheel asset to release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        with:
+          upload_url: ${{steps.create_gh_release.outputs.upload_url}}
+          asset_path: ${{env.DJ_WHEEL_PATH}}
+          asset_name: pip-datajoint-${{env.DJ_VERSION}}.whl
+          asset_content_type: application/zip
+      - name: Upload pip sdist asset to release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        with:
+          upload_url: ${{steps.create_gh_release.outputs.upload_url}}
+          asset_path: ${{env.DJ_SDIST_PATH}}
+          asset_name: pip-datajoint-${{env.DJ_VERSION}}.tar.gz
+          asset_content_type: application/gzip
+      - name: Publish pip release
+        run: |
+          export HOST_UID=$(id -u)
+          docker-compose -f docker-compose-build.yaml run \
+            -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} app \
+            sh -lc "pip install twine && python -m twine upload dist/*"
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{secrets.docker_username}}
+          password: ${{secrets.docker_password}}
+      - name: Publish image
+        run: |
+          IMAGE=$(docker images --filter "reference=datajoint/datajoint*" --format "{{.Repository}}")
+          TAG=$(docker images --filter "reference=datajoint/datajoint*" --format "{{.Tag}}")
+          docker push "${IMAGE}:${TAG}"
+          docker tag "${IMAGE}:${TAG}" "${IMAGE}:${TAG}-${GITHUB_SHA:0:7}"
+          docker push "${IMAGE}:${TAG}-${GITHUB_SHA:0:7}"
+          [ "$PY_VER" == "3.9" ] && [ "$DISTRO" == "debian" ] \
+                  && docker tag "${IMAGE}:${TAG}" "${IMAGE}:latest" \
+                  && docker push "${IMAGE}:latest" \
+              || echo "skipping 'latest' tag..."

.gitignore

@@ -24,5 +24,6 @@ notebook
 .vscode
 __main__.py
 jupyter_custom.js
-apk_requirements.txt
 .eggs
+*.code-workspace
+docs/site

CHANGELOG.md

@@ -1,9 +1,32 @@
 ## Release notes
 
-### 0.13.5 -- TBA
-* Update - Switch testing image from `pydev` to `djtest` PR #1012
-
-### 0.13.4 -- March, 28 2022
+### 0.14.0 -- TBA
+* Bugfix - Activating a schema requires all tables to exist even if `create_tables=False` PR [#1058](https://github.com/datajoint/datajoint-python/pull/1058)
+* Update - Populate call with `reserve_jobs=True` to exclude `error` and `ignore` keys - PR [#1062](https://github.com/datajoint/datajoint-python/pull/1062)
+* Add - Support for inserting data with CSV files - PR [#1067](https://github.com/datajoint/datajoint-python/pull/1067)
+* Update - Switch testing image from `pydev` to `djtest` PR [#1012](https://github.com/datajoint/datajoint-python/pull/1012)
+
+### 0.13.8 -- Sep 21, 2022
+* Add - New documentation structure based on markdown PR [#1052](https://github.com/datajoint/datajoint-python/pull/1052)
+* Bugfix - Fix queries with backslashes ([#999](https://github.com/datajoint/datajoint-python/issues/999)) PR [#1052](https://github.com/datajoint/datajoint-python/pull/1052)
+
+### 0.13.7 -- Jul 13, 2022
+* Bugfix - Fix networkx incompatable change by version pinning to 2.6.3 (#1035) PR #1036
+* Add - Support for serializing numpy datetime64 types (#1022) PR #1036
+* Update - Add traceback to default logging PR #1036
+
+### 0.13.6 -- Jun 13, 2022
+* Add - Config option to set threshold for when to stop using checksums for filepath stores. PR #1025
+* Add - Unified package level logger for package (#667) PR #1031
+* Update - Swap various datajoint messages, warnings, etc. to use the new logger. (#667) PR #1031
+* Bugfix - Fix query caching deleting non-datajoint files PR #1027
+* Update - Minimum Python version for Datajoint-Python is now 3.7 PR #1027
+
+### 0.13.5 -- May 19, 2022
+* Update - Import ABC from collections.abc for Python 3.10 compatibility
+* Bugfix - Fix multiprocessing value error (#1013) PR #1026
+
+### 0.13.4 -- Mar, 28 2022
 * Add - Allow reading blobs produced by legacy 32-bit compiled mYm library for matlab. PR #995
 * Bugfix - Add missing `jobs` argument for multiprocessing PR #997
 * Add - Test for multiprocessing PR #1008
@@ -253,7 +276,7 @@ Documentation and tutorials available at https://docs.datajoint.io and https://t
 
 ### 0.3.4
 * Added method the `ERD.add_parts` method, which adds the part tables of all tables currently in the ERD.
-* `ERD() + arg` and `ERD() - arg` can now accept relation classes as arg.
+* `ERD() + arg` and `ERD() - arg` can now accept table classes as arg.
 
 ### 0.3.3
 * Suppressed warnings (redirected them to logging).  Previoiusly, scipy would throw warnings in ERD, for example.
@@ -263,5 +286,5 @@ Documentation and tutorials available at https://docs.datajoint.io and https://t
 
 ### 0.3.2.
 * Fixed issue #223:  `insert` can insert relations without fetching.
-* ERD() now takes the `context` argument, which specifies in which context to look for classes. The default is taken from the argument (schema or relation).
+* ERD() now takes the `context` argument, which specifies in which context to look for classes. The default is taken from the argument (schema or table).
 * ERD.draw() no longer has the `prefix` argument: class names are shown as found in the context.

Dockerfile

@@ -1,5 +1,9 @@
-FROM datajoint/djbase
-
-COPY --chown=anaconda:anaconda . /tmp/src
-RUN pip install --no-cache-dir /tmp/src && \
-    rm -rf /tmp/src
+ARG IMAGE=djbase
+ARG PY_VER=3.9
+ARG DISTRO=debian
+FROM datajoint/${IMAGE}:py${PY_VER}-${DISTRO}
+COPY --chown=anaconda:anaconda ./setup.py ./datajoint.pub ./requirements.txt /main/
+COPY --chown=anaconda:anaconda ./datajoint /main/datajoint
+RUN \
+    pip install --no-cache-dir /main && \
+    rm -r /main/*

LNX-docker-compose.yml

@@ -1,12 +1,12 @@
-# docker-compose -f LNX-docker-compose.yml --env-file LNX.env up --exit-code-from app --build
-version: '2.2'
+# docker compose -f LNX-docker-compose.yml --env-file LNX.env up --exit-code-from app --build
+version: '2.4'
 x-net: &net
   networks:
       - main
 services:
   db:
     <<: *net
-    image: datajoint/mysql:$MYSQL_VER
+    image: datajoint/mysql:${MYSQL_VER}
     environment:
       - MYSQL_ROOT_PASSWORD=simple
     # ports:
@@ -15,7 +15,7 @@ services:
     #   - ./mysql/data:/var/lib/mysql
   minio:
     <<: *net
-    image: minio/minio:$MINIO_VER
+    image: minio/minio:${MINIO_VER}
     environment:
       - MINIO_ACCESS_KEY=datajoint
       - MINIO_SECRET_KEY=datajoint
@@ -32,7 +32,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: datajoint/nginx:v0.2.0
+    image: datajoint/nginx:v0.2.4
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306
@@ -77,16 +77,14 @@ services:
         set -e
         pip install --user nose nose-cov
         pip install -e .
-        pip freeze | grep datajoint
+        pip list --format=freeze | grep datajoint
         nosetests -vsw tests --with-coverage --cover-package=datajoint
-        # jupyter notebook
     # ports:
     #   - "8888:8888"
     user: ${HOST_UID}:anaconda
     volumes:
       - .:/src
       - /tmp/.X11-unix:/tmp/.X11-unix:rw
-      # - ./apk_requirements.txt:/tmp/apk_requirements.txt
       # - ./notebooks:/home/dja/notebooks
 networks:
   main: