DataJoint 2.0

.devcontainer/Dockerfile

@@ -8,6 +8,7 @@ RUN \
     pip uninstall datajoint -y
 
 USER root
-ENV DJ_HOST db
-ENV DJ_USER root
-ENV DJ_PASS password
+ENV DJ_HOST=db
+ENV DJ_USER=root
+ENV DJ_PASS=password
+ENV S3_ENDPOINT=minio:9000

.devcontainer/devcontainer.json

@@ -1,56 +1,6 @@
-// For format details, see https://aka.ms/devcontainer.json. For config options, see the
-// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose
 {
-	"name": "Existing Docker Compose (Extend)",
-	// Update the 'dockerComposeFile' list if you have more compose files or use different names.
-	// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
-	"dockerComposeFile": [
-		"../docker-compose.yaml",
-		"docker-compose.yml"
-	],
-	// The 'service' property is the name of the service for the container that VS Code should
-	// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
-	"service": "app",
-	// The optional 'workspaceFolder' property is the path VS Code should open by default when
-	// connected. This is typically a file mount in .devcontainer/docker-compose.yml
-	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
-	// Features to add to the dev container. More info: https://containers.dev/features.
-	// "features": {},
-	// Use 'forwardPorts' to make a list of ports inside the container available locally.
-	"forwardPorts": [
-		80,
-		443,
-		3306,
-		8080,
-		9000
-	],
-	"mounts": [
-	  "type=bind,source=${env:SSH_AUTH_SOCK},target=/ssh-agent"
-	],
-	"containerEnv": {
-	  "SSH_AUTH_SOCK": "/ssh-agent"
-	},
-	// Uncomment the next line if you want start specific services in your Docker Compose config.
-	// "runServices": [],
-	// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
-	"shutdownAction": "stopCompose",
-	"onCreateCommand": "python3 -m pip install -q -e .[dev]",
-	"features": {
-		"ghcr.io/devcontainers/features/git:1": {},
-		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
-		"ghcr.io/devcontainers/features/github-cli:1": {},
-	},
-	// Configure tool-specific properties.
-	"customizations": {
-		"vscode": {
-			"extensions": [
-				"ms-python.python"
-			]
-		}
-	},
-	"remoteEnv": {
-		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
-	}
-	// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
-	// "remoteUser": "devcontainer"
+  "dockerComposeFile": ["../docker-compose.yaml", "docker-compose.yml"],
+  "service": "app",
+  "workspaceFolder": "/src",
+  "postCreateCommand": "curl -fsSL https://pixi.sh/install.sh | bash && echo 'export PATH=\"$HOME/.pixi/bin:$PATH\"' >> ~/.bashrc"
 }

.devcontainer/docker-compose.yml

@@ -1,30 +1,14 @@
+# Devcontainer overrides for the app service from ../docker-compose.yaml
+# Inherits db and minio services automatically
 services:
-  # Update this to the name of the service you want to work with in your docker-compose.yml file
   app:
-    # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer
-    # folder. Note that the path of the Dockerfile and context is relative to the *primary*
-    # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile"
-    # array). The sample below assumes your primary file is in the root of your project.
     container_name: datajoint-python-devcontainer
-    image: datajoint/datajoint-python-devcontainer:${PY_VER:-3.11}-${DISTRO:-bookworm}
     build:
-      context: .
+      context: ..
       dockerfile: .devcontainer/Dockerfile
       args:
         - PY_VER=${PY_VER:-3.11}
         - DISTRO=${DISTRO:-bookworm}
-
-    volumes:
-      # Update this to wherever you want VS Code to mount the folder of your project
-      - ..:/workspaces:cached
-
-    # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.
-    # cap_add:
-    #   - SYS_PTRACE
-    # security_opt:
-    #   - seccomp:unconfined
-
     user: root
-
-    # Overrides default command so things don't shut down after the process ends.
+    # Keep container running for devcontainer
     command: /bin/sh -c "while sleep 1000; do :; done"

.gitattributes

@@ -0,0 +1,2 @@
+# SCM syntax highlighting & preventing 3-way merges
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true

.github/pr_labeler.yaml

@@ -1,8 +1,8 @@
 # https://github.com/actions/labeler
 breaking:
-- head-branch: ['breaking', 'BREAKING']
+- head-branch: ['breaking', 'BREAKING', 'pre/v2.0']
 bug:
-- head-branch: ['fix', 'FIX', 'bug', 'BUG']
+- head-branch: ['fix', 'FIX', 'bug', 'BUG', 'pre/v2.0']
 feature:
 - head-branch: ['feat', 'FEAT']
 documentation:

.github/workflows/label_prs.yaml

@@ -14,5 +14,5 @@ jobs:
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         configuration-path: .github/pr_labeler.yaml
-        sync-labels: true
+        sync-labels: false  # Don't remove manually added labels
         dot: true

.github/workflows/lint.yaml

@@ -23,7 +23,7 @@ jobs:
         extra_args: codespell --all-files
     - uses: pre-commit/[email protected]
       with:
-        extra_args: black --all-files
+        extra_args: ruff --all-files
     - uses: pre-commit/[email protected]
       with:
-        extra_args: flake8 --all-files
+        extra_args: ruff-format --all-files

.github/workflows/post_draft_release_published.yaml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - py_ver: "3.9"
+          - py_ver: "3.10"
     runs-on: ubuntu-latest
     env:
       PY_VER: ${{matrix.py_ver}}
@@ -40,14 +40,14 @@ jobs:
       - name: Update version.py
         run: |
           VERSION=$(echo "${{ github.event.release.name }}" | grep -oP '\d+\.\d+\.\d+')
-          sed -i "s/^__version__ = .*/__version__ = \"$VERSION\"/" datajoint/version.py
-          cat datajoint/version.py
+          sed -i "s/^__version__ = .*/__version__ = \"$VERSION\"/" src/datajoint/version.py
+          cat src/datajoint/version.py
           # Commit the changes
           BRANCH_NAME="update-version-$VERSION"
           git switch -c $BRANCH_NAME
           git config --global user.name "github-actions"
           git config --global user.email "[email protected]"
-          git add datajoint/version.py
+          git add src/datajoint/version.py
           git commit -m "Update version.py to $VERSION"
           echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
       - name: Update README.md badge

.github/workflows/test.yaml

@@ -1,42 +1,55 @@
 name: Test
+
 on:
   push:
     branches:
-      - "**" # every branch
-      - "!gh-pages" # exclude gh-pages branch
-      - "!stage*" # exclude branches beginning with stage
+      - "**"
+      - "!gh-pages"
+      - "!stage*"
     paths:
-      - "datajoint"
-      - "tests"
+      - "src/datajoint/**"
+      - "tests/**"
+      - "pyproject.toml"
+      - "pixi.lock"
+      - ".github/workflows/test.yaml"
   pull_request:
     branches:
-      - "**" # every branch
-      - "!gh-pages" # exclude gh-pages branch
-      - "!stage*" # exclude branches beginning with stage
+      - "**"
+      - "!gh-pages"
+      - "!stage*"
     paths:
-      - "datajoint"
-      - "tests"
+      - "src/datajoint/**"
+      - "tests/**"
+      - "pyproject.toml"
+      - "pixi.lock"
+      - ".github/workflows/test.yaml"
+
 jobs:
   test:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        py_ver: ["3.9", "3.10", "3.11", "3.12", "3.13"]
-        mysql_ver: ["8.0"]
-        include:
-          - py_ver: "3.9"
-            mysql_ver: "5.7"
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python ${{matrix.py_ver}}
-        uses: actions/setup-python@v5
+
+      - name: Set up pixi
+        uses: prefix-dev/[email protected]
         with:
-          python-version: ${{matrix.py_ver}}
-      - name: Integration test
-        env:
-          PY_VER: ${{matrix.py_ver}}
-          MYSQL_VER: ${{matrix.mysql_ver}}
-          # taking default variables set in docker-compose.yaml to sync with local test
-        run: |
-          export HOST_UID=$(id -u)
-          docker compose --profile test up --quiet-pull --build --exit-code-from djtest djtest
+          cache: true
+          locked: false
+
+      - name: Run tests
+        run: pixi run -e test test-cov
+
+  # Unit tests run without containers (faster feedback)
+  unit-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up pixi
+        uses: prefix-dev/[email protected]
+        with:
+          cache: true
+          locked: false
+
+      - name: Run unit tests
+        run: pixi run -e test pytest tests/unit -v

.gitignore

@@ -185,3 +185,10 @@ cython_debug/
 dj_local_conf.json
 *.env
 !.vscode/launch.json
+# pixi environments
+.pixi
+_content/
+
+# Local config
+.secrets/
+datajoint.json

.pre-commit-config.yaml

@@ -1,8 +1,7 @@
-# pip install datajoint[test]
 # pre-commit install
 # pre-commit run --all-files
 # pre-commit autoupdate
-# SKIP=flake8 git commit -m "foo"
+# SKIP=ruff git commit -m "foo"
 
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
@@ -20,43 +19,40 @@ repos:
   rev: v2.4.1
   hooks:
     - id: codespell
-- repo: https://github.com/pycqa/isort
-  rev: 6.0.1  # Use the latest stable version
+      args: [--toml, pyproject.toml]
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.8.4
   hooks:
-    - id: isort
-      args:
-        - --profile=black  # Optional, makes isort compatible with Black
-- repo: https://github.com/psf/black
-  rev: 25.1.0 # matching versions in pyproject.toml and github actions
-  hooks:
-  - id: black
-    args: ["--check", "-v", "datajoint", "tests", "--diff"] # --required-version is conflicting with pre-commit
-- repo: https://github.com/PyCQA/flake8
-  rev: 7.3.0
-  hooks:
-  # syntax tests
-  - id: flake8
-    args:
-      - --select=E9,F63,F7,F82
-      - --count
-      - --show-source
-      - --statistics
-    files: datajoint # a lot of files in tests are not compliant
-  # style tests
-  - id: flake8
-    args:
-      - --ignore=E203,E722,W503
-      - --count
-      - --max-complexity=62
-      - --max-line-length=127
-      - --statistics
-      - --per-file-ignores=datajoint/diagram.py:C901
-    files: datajoint # a lot of files in tests are not compliant
+    # Run the linter
+    - id: ruff
+      args: [--fix]
+      files: ^(src/|tests/)
+    # Run the formatter
+    - id: ruff-format
+      files: ^(src/|tests/)
 - repo: https://github.com/rhysd/actionlint
   rev: v1.7.7
   hooks:
   # lint github actions workflow yaml
   - id: actionlint
-
-## Suggest to add pytest hook that runs unit test | Prerequisite: split unit/integration test
-## https://github.com/datajoint/datajoint-python/issues/1211
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.14.1
+  hooks:
+    - id: mypy
+      files: ^src/datajoint/
+      additional_dependencies:
+        - pydantic
+        - pydantic-settings
+        - types-PyMySQL
+        - types-tqdm
+        - pandas-stubs
+        - numpy
+- repo: local
+  hooks:
+    - id: unit-tests
+      name: unit tests
+      entry: pytest tests/unit/ -v --tb=short
+      language: system
+      pass_filenames: false
+      always_run: true
+      stages: [pre-commit]