Merge branch 'feat/pytest-container-management' of https://github.com/d-v-b/datajoint-python into feat/djzarr

Author: d-v-b

.gitattributes

@@ -0,0 +1,2 @@
+# SCM syntax highlighting & preventing 3-way merges
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true

.gitignore

@@ -185,3 +185,5 @@ cython_debug/
 dj_local_conf.json
 *.env
 !.vscode/launch.json
+# pixi environments
+.pixi

pixi.lock

@@ -84,8 +84,10 @@ datajoint = "datajoint.cli:cli"
 test = [
   "pytest",
   "pytest-cov",
+  "pytest-env",
   "docker",
   "requests",
+  "graphviz"
 ]
 
 [project.optional-dependencies]
@@ -109,3 +111,33 @@ package-dir = {"" = "src"}
 
 [tool.setuptools.dynamic]
 version = { attr = "datajoint.version.__version__"}
+
+[tool.pytest_env]
+# Default values - pytest fixtures will override with actual container details
+DJ_USER="root"
+DJ_PASS="password"
+DJ_TEST_USER="datajoint"
+DJ_TEST_PASSWORD="datajoint"
+S3_ACCESS_KEY="datajoint"
+S3_SECRET_KEY="datajoint"
+S3_BUCKET="datajoint.test"
+PYTHON_USER="dja"
+JUPYTER_PASSWORD="datajoint"
+
+
+[tool.pixi.workspace]
+channels = ["conda-forge"]
+platforms = ["linux-64"]
+
+[tool.pixi.pypi-dependencies]
+datajoint = { path = ".", editable = true }
+
+[tool.pixi.environments]
+default = { solve-group = "default" }
+dev = { features = ["dev"], solve-group = "default" }
+test = { features = ["test"], solve-group = "default" }
+
+[tool.pixi.tasks]
+
+[tool.pixi.dependencies]
+graphviz = ">=13.1.2,<14"

pyproject.toml

@@ -318,6 +318,7 @@ def __setitem__(self, key, value):
             "database.host",
             "database.user",
             "database.password",
+            "database.port",
             "external.aws_access_key_id",
             "external.aws_secret_access_key",
             "loglevel",
@@ -328,6 +329,7 @@ def __setitem__(self, key, value):
                 "DJ_HOST",
                 "DJ_USER",
                 "DJ_PASS",
+                "DJ_PORT",
                 "DJ_AWS_ACCESS_KEY_ID",
                 "DJ_AWS_SECRET_ACCESS_KEY",
                 "DJ_LOG_LEVEL",
@@ -336,6 +338,14 @@ def __setitem__(self, key, value):
     )
     if v is not None
 }
+
+# Convert DJ_PORT from string to int if present
+if "database.port" in mapping and mapping["database.port"] is not None:
+    try:
+        mapping["database.port"] = int(mapping["database.port"])
+    except ValueError:
+        logger.warning(f"Invalid DJ_PORT value: {mapping['database.port']}, using default port 3306")
+        del mapping["database.port"]
 if mapping:
     logger.info(f"Overloaded settings {tuple(mapping)} from environment variables.")
     config.update(mapping)

src/datajoint/settings.py

@@ -34,6 +34,18 @@
 logger = logging.getLogger(__name__)
 
 
+def pytest_sessionstart(session):
+    """Called after the Session object has been created and configured."""
+    # This runs very early, before most fixtures, but we don't have container info yet
+    pass
+
+
+def pytest_configure(config):
+    """Called after command line options have been parsed."""
+    # This runs before pytest_sessionstart but still too early for containers
+    pass
+
+
 
 
 # Global container registry for cleanup
@@ -298,13 +310,42 @@ def enable_filepath_feature(monkeypatch):
 @pytest.fixture(scope="session")
 def db_creds_test(mysql_container) -> Dict:
     _, host, port = mysql_container
+    # Set environment variables for DataJoint at module level
+    os.environ["DJ_TEST_HOST"] = host
+    os.environ["DJ_TEST_PORT"] = str(port)
+
+    # Also update DataJoint's test configuration directly
+    dj.config["database.test.host"] = host
+    dj.config["database.test.port"] = port
+
     return dict(
         host=f"{host}:{port}",
         user=os.getenv("DJ_TEST_USER", "datajoint"),
         password=os.getenv("DJ_TEST_PASSWORD", "datajoint"),
     )
 
 
+@pytest.fixture(scope="session", autouse=True)
+def configure_datajoint_for_containers(mysql_container):
+    """Configure DataJoint to use pytest-managed containers. Runs automatically for all tests."""
+    _, host, port = mysql_container
+
+    # Set environment variables FIRST - these will be inherited by subprocesses
+    logger.info(f"🔧 Setting environment: DJ_HOST={host}, DJ_PORT={port}")
+    os.environ["DJ_HOST"] = host
+    os.environ["DJ_PORT"] = str(port)
+
+    # Verify the environment variables were set
+    logger.info(f"🔧 Environment after setting: DJ_HOST={os.environ.get('DJ_HOST')}, DJ_PORT={os.environ.get('DJ_PORT')}")
+
+    # Also update DataJoint's configuration directly for in-process connections
+    dj.config["database.host"] = host
+    dj.config["database.port"] = port
+
+    logger.info(f"🔧 Configured DataJoint to use MySQL container at {host}:{port}")
+    return host, port  # Return values so other fixtures can use them
+
+
 @pytest.fixture(scope="session")
 def db_creds_root(mysql_container) -> Dict:
     _, host, port = mysql_container
@@ -434,6 +475,8 @@ def connection_test(connection_root, prefix, db_creds_test):
 @pytest.fixture(scope="session")
 def s3_creds(minio_container) -> Dict:
     _, host, port = minio_container
+    # Set environment variable for S3 endpoint at module level
+    os.environ["S3_ENDPOINT"] = f"{host}:{port}"
     return dict(
         endpoint=f"{host}:{port}",
         access_key=os.environ.get("S3_ACCESS_KEY", "datajoint"),