use pytest to manage docker container startup for tests

.github/workflows/test.yaml

@@ -34,9 +34,7 @@ jobs:
           python-version: ${{matrix.py_ver}}
       - name: Integration test
         env:
-          PY_VER: ${{matrix.py_ver}}
           MYSQL_VER: ${{matrix.mysql_ver}}
-          # taking default variables set in docker-compose.yaml to sync with local test
         run: |
-          export HOST_UID=$(id -u)
-          docker compose --profile test up --quiet-pull --build --exit-code-from djtest djtest
+          pip install -e ".[test]"
+          pytest --cov-report term-missing --cov=datajoint tests

docker-compose.yaml

@@ -1,4 +1,5 @@
-# HOST_UID=$(id -u) PY_VER=3.11 DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) docker compose --profile test up --build --exit-code-from djtest djtest
+# Development environment with MySQL and MinIO services
+# To run tests: pytest --cov-report term-missing --cov=datajoint tests
 services:
   db:
     image: datajoint/mysql:${MYSQL_VER:-8.0}

pyproject.toml

@@ -22,7 +22,7 @@ dependencies = [
   "urllib3",
   "setuptools",
 ]
-requires-python = ">=3.9,<4.0"
+requires-python = ">=3.11,<4.0"
 authors = [
   {name = "Dimitri Yatsenko", email = "[email protected]"},
   {name = "Thinh Nguyen", email = "[email protected]"},
@@ -78,11 +78,16 @@ Repository = "https://github.com/datajoint/datajoint-python"
 dj = "datajoint.cli:cli"
 datajoint = "datajoint.cli:cli"
 
-[project.optional-dependencies]
+[dependency-groups]
 test = [
   "pytest",
   "pytest-cov",
+  "pytest-env",
+  "docker",
+  "requests",
 ]
+
+[project.optional-dependencies]
 dev = [
   "pre-commit",
   "black==24.2.0",
@@ -103,3 +108,15 @@ package-dir = {"" = "src"}
 
 [tool.setuptools.dynamic]
 version = { attr = "datajoint.version.__version__"}
+
+[tool.pytest_env]
+# Default values - pytest fixtures will override with actual container details
+DJ_USER="root"
+DJ_PASS="password"
+DJ_TEST_USER="datajoint"
+DJ_TEST_PASSWORD="datajoint"
+S3_ACCESS_KEY="datajoint"
+S3_SECRET_KEY="datajoint"
+S3_BUCKET="datajoint.test"
+PYTHON_USER="dja"
+JUPYTER_PASSWORD="datajoint"

src/datajoint/settings.py

@@ -278,6 +278,7 @@ def __setitem__(self, key, value):
             "database.host",
             "database.user",
             "database.password",
+            "database.port",
             "external.aws_access_key_id",
             "external.aws_secret_access_key",
             "loglevel",
@@ -288,6 +289,7 @@ def __setitem__(self, key, value):
                 "DJ_HOST",
                 "DJ_USER",
                 "DJ_PASS",
+                "DJ_PORT",
                 "DJ_AWS_ACCESS_KEY_ID",
                 "DJ_AWS_SECRET_ACCESS_KEY",
                 "DJ_LOG_LEVEL",
@@ -296,6 +298,14 @@ def __setitem__(self, key, value):
     )
     if v is not None
 }
+
+# Convert DJ_PORT from string to int if present
+if "database.port" in mapping and mapping["database.port"] is not None:
+    try:
+        mapping["database.port"] = int(mapping["database.port"])
+    except ValueError:
+        logger.warning(f"Invalid DJ_PORT value: {mapping['database.port']}, using default port 3306")
+        del mapping["database.port"]
 if mapping:
     logger.info(f"Overloaded settings {tuple(mapping)} from environment variables.")
     config.update(mapping)