Fix test failures and update to new type system

Source code fixes:
- Add download_path setting and squeeze handling in fetch.py
- Add filename collision handling in AttachType and XAttachType
- Fix is_blob detection to check both BLOB and NATIVE_BLOB patterns
- Fix FilepathType.validate to accept Path objects
- Add proper error message for undecorated tables

Test infrastructure updates:
- Update schema_external.py to use new <xblob@store>, <xattach@store>, <filepath@store> syntax
- Update all test tables to use <djblob> instead of longblob for serialization
- Configure object_storage.stores in conftest.py fixtures
- Remove obsolete test_admin.py (set_password was removed)
- Fix connection passing in various tests to avoid credential prompts
- Fix test_query_caching to handle existing directories

README:
- Add Developer Guide section with setup, test, and pre-commit instructions

Test results: 408 passed, 2 skipped (macOS multiprocessing limitation)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <[email protected]>

Author: dimitri-yatsenko

README.md

@@ -141,3 +141,66 @@ DataJoint (<https://datajoint.com>).
   - [Contribution Guidelines](https://docs.datajoint.com/about/contribute/)
 
   - [Developer Guide](https://docs.datajoint.com/core/datajoint-python/latest/develop/)
+
+## Developer Guide
+
+### Prerequisites
+
+- [Docker](https://docs.docker.com/get-docker/) for running MySQL and MinIO services
+- [pixi](https://prefix.dev/docs/pixi/overview) package manager (or pip/conda)
+
+### Setting Up the Development Environment
+
+1. Clone the repository and install dependencies:
+
+   ```bash
+   git clone https://github.com/datajoint/datajoint-python.git
+   cd datajoint-python
+   pixi install
+   ```
+
+2. Start the required services (MySQL and MinIO):
+
+   ```bash
+   docker compose up -d db minio
+   ```
+
+### Running Tests
+
+Run tests with pytest using the test environment:
+
+```bash
+DJ_HOST=localhost DJ_PORT=3306 S3_ENDPOINT=localhost:9000 python -m pytest tests/
+```
+
+Or run specific test files:
+
+```bash
+DJ_HOST=localhost DJ_PORT=3306 S3_ENDPOINT=localhost:9000 python -m pytest tests/test_blob.py -v
+```
+
+### Running Pre-commit Checks
+
+Pre-commit hooks ensure code quality before commits. Install and run them:
+
+```bash
+# Install pre-commit hooks
+pre-commit install
+
+# Run all pre-commit checks manually
+pre-commit run --all-files
+
+# Run specific hooks
+pre-commit run ruff --all-files
+pre-commit run mypy --all-files
+```
+
+### Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `DJ_HOST` | `localhost` | MySQL server hostname |
+| `DJ_PORT` | `3306` | MySQL server port |
+| `DJ_USER` | `datajoint` | MySQL username |
+| `DJ_PASS` | `datajoint` | MySQL password |
+| `S3_ENDPOINT` | `localhost:9000` | MinIO/S3 endpoint |

src/datajoint/builtin_types.py

@@ -523,7 +523,22 @@ def decode(self, stored: bytes, *, key: dict | None = None) -> str:
         download_path.mkdir(parents=True, exist_ok=True)
         local_path = download_path / filename
 
-        local_path.write_bytes(contents)
+        # Handle filename collision - if file exists with different content, add suffix
+        if local_path.exists():
+            existing_contents = local_path.read_bytes()
+            if existing_contents != contents:
+                # Find unique filename
+                stem = local_path.stem
+                suffix = local_path.suffix
+                counter = 1
+                while local_path.exists() and local_path.read_bytes() != contents:
+                    local_path = download_path / f"{stem}_{counter}{suffix}"
+                    counter += 1
+
+        # Only write if file doesn't exist or has different content
+        if not local_path.exists():
+            local_path.write_bytes(contents)
+
         return str(local_path)
 
     def validate(self, value: Any) -> None:
@@ -626,7 +641,22 @@ def decode(self, stored: bytes, *, key: dict | None = None) -> str:
         download_path.mkdir(parents=True, exist_ok=True)
         local_path = download_path / filename
 
-        local_path.write_bytes(contents)
+        # Handle filename collision - if file exists with different content, add suffix
+        if local_path.exists():
+            existing_contents = local_path.read_bytes()
+            if existing_contents != contents:
+                # Find unique filename
+                stem = local_path.stem
+                suffix = local_path.suffix
+                counter = 1
+                while local_path.exists() and local_path.read_bytes() != contents:
+                    local_path = download_path / f"{stem}_{counter}{suffix}"
+                    counter += 1
+
+        # Only write if file doesn't exist or has different content
+        if not local_path.exists():
+            local_path.write_bytes(contents)
+
         return str(local_path)
 
     def validate(self, value: Any) -> None:
@@ -741,6 +771,8 @@ def decode(self, stored: dict, *, key: dict | None = None) -> Any:
         return ObjectRef.from_json(stored, backend=backend)
 
     def validate(self, value: Any) -> None:
-        """Validate that value is a path string."""
-        if not isinstance(value, str):
-            raise TypeError(f"<filepath> expects a path string, got {type(value).__name__}")
+        """Validate that value is a path string or Path object."""
+        from pathlib import Path
+
+        if not isinstance(value, (str, Path)):
+            raise TypeError(f"<filepath> expects a path string or Path, got {type(value).__name__}")

src/datajoint/fetch.py

@@ -49,9 +49,11 @@ def _get(connection, attr, data, squeeze, download_path):
     :param attr: attribute from the table's heading
     :param data: raw value fetched from the database
     :param squeeze: if True squeeze blobs (legacy, unused)
-    :param download_path: for fetches that download data (legacy, unused in simplified model)
+    :param download_path: for fetches that download data (attachments, filepaths)
     :return: decoded data
     """
+    from .settings import config
+
     if data is None:
         return None
 
@@ -69,9 +71,19 @@ def _get(connection, attr, data, squeeze, download_path):
         elif final_dtype.lower() == "binary(16)":
             data = uuid_module.UUID(bytes=data)
 
-        # Apply decoders in reverse order: innermost first, then outermost
-        for attr_type in reversed(type_chain):
-            data = attr_type.decode(data, key=None)
+        # Temporarily set download_path for types that need it (attachments, filepaths)
+        original_download_path = config.get("download_path", ".")
+        config["download_path"] = str(download_path)
+        try:
+            # Apply decoders in reverse order: innermost first, then outermost
+            for attr_type in reversed(type_chain):
+                data = attr_type.decode(data, key=None)
+        finally:
+            config["download_path"] = original_download_path
+
+        # Apply squeeze for blob types (removes singleton dimensions from arrays)
+        if squeeze and isinstance(data, np.ndarray):
+            data = data.squeeze()
 
         return data
 

src/datajoint/heading.py

@@ -332,8 +332,8 @@ def _init_from_database(self):
                     attr["type"] = attr["adapter"].dtype
                     if not any(r.match(attr["type"]) for r in TYPE_PATTERN.values()):
                         raise DataJointError(f"Invalid dtype '{attr['type']}' in attribute type <{adapter_name}>.")
-                    # Update is_blob based on resolved dtype
-                    attr["is_blob"] = bool(TYPE_PATTERN["BLOB"].match(attr["type"]))
+                    # Update is_blob based on resolved dtype (check both BLOB and NATIVE_BLOB patterns)
+                    attr["is_blob"] = any(TYPE_PATTERN[t].match(attr["type"]) for t in ("BLOB", "NATIVE_BLOB"))
 
             # Handle core type aliases (uuid, float32, etc.)
             if special:

src/datajoint/settings.py

@@ -267,6 +267,9 @@ class Config(BaseSettings):
     cache: Path | None = None
     query_cache: Path | None = None
 
+    # Download path for attachments and filepaths
+    download_path: str = "."
+
     # Internal: track where config was loaded from
     _config_path: Path | None = None
     _secrets_dir: Path | None = None
@@ -675,6 +678,29 @@ def __setitem__(self, key: str, value: Any) -> None:
                 obj = getattr(obj, part)
             setattr(obj, parts[-1], value)
 
+    def __delitem__(self, key: str) -> None:
+        """Reset setting to default by dot-notation key."""
+        # Get the default value from the model fields
+        parts = key.split(".")
+        if len(parts) == 1:
+            field_info = self.model_fields.get(key)
+            if field_info is not None:
+                default = field_info.default
+                if default is not None:
+                    setattr(self, key, default)
+                elif field_info.default_factory is not None:
+                    setattr(self, key, field_info.default_factory())
+                else:
+                    setattr(self, key, None)
+            else:
+                raise KeyError(f"Setting '{key}' not found")
+        else:
+            # For nested settings, reset to None or empty
+            obj: Any = self
+            for part in parts[:-1]:
+                obj = getattr(obj, part)
+            setattr(obj, parts[-1], None)
+
     def get(self, key: str, default: Any = None) -> Any:
         """Get setting with optional default value."""
         try:

src/datajoint/table.py

@@ -254,6 +254,11 @@ def full_table_name(self):
         """
         :return: full table name in the schema
         """
+        if self.database is None or self.table_name is None:
+            raise DataJointError(
+                f"Class {self.__class__.__name__} is not associated with a schema. "
+                "Apply a schema decorator or use schema() to bind it."
+            )
         return r"`{0:s}`.`{1:s}`".format(self.database, self.table_name)
 
     @property

tests/conftest.py

@@ -201,33 +201,56 @@ def connection_test(connection_root, prefix, db_creds_test):
 
 @pytest.fixture(scope="session")
 def stores_config(s3_creds, tmpdir_factory):
+    """Configure object storage stores for tests."""
     return {
-        "raw": dict(protocol="file", location=tmpdir_factory.mktemp("raw")),
+        "raw": dict(protocol="file", location=str(tmpdir_factory.mktemp("raw"))),
         "repo": dict(
-            stage=tmpdir_factory.mktemp("repo"),
+            stage=str(tmpdir_factory.mktemp("repo")),
             protocol="file",
-            location=tmpdir_factory.mktemp("repo"),
+            location=str(tmpdir_factory.mktemp("repo")),
         ),
         "repo-s3": dict(
-            s3_creds,
             protocol="s3",
+            endpoint=s3_creds["endpoint"],
+            access_key=s3_creds["access_key"],
+            secret_key=s3_creds["secret_key"],
+            bucket=s3_creds.get("bucket", "datajoint-test"),
             location="dj/repo",
-            stage=tmpdir_factory.mktemp("repo-s3"),
+            stage=str(tmpdir_factory.mktemp("repo-s3")),
+            secure=False,  # MinIO runs without SSL in tests
+        ),
+        "local": dict(protocol="file", location=str(tmpdir_factory.mktemp("local"))),
+        "share": dict(
+            protocol="s3",
+            endpoint=s3_creds["endpoint"],
+            access_key=s3_creds["access_key"],
+            secret_key=s3_creds["secret_key"],
+            bucket=s3_creds.get("bucket", "datajoint-test"),
+            location="dj/store/repo",
+            secure=False,  # MinIO runs without SSL in tests
         ),
-        "local": dict(protocol="file", location=tmpdir_factory.mktemp("local"), subfolding=(1, 1)),
-        "share": dict(s3_creds, protocol="s3", location="dj/store/repo", subfolding=(2, 4)),
     }
 
 
 @pytest.fixture
 def mock_stores(stores_config):
-    og_stores_config = dj.config.get("stores")
-    dj.config["stores"] = stores_config
+    """Configure object storage stores for tests using new object_storage system."""
+    # Save original configuration
+    og_project_name = dj.config.object_storage.project_name
+    og_stores = dict(dj.config.object_storage.stores)
+
+    # Set test configuration
+    dj.config.object_storage.project_name = "djtest"
+    dj.config.object_storage.stores.clear()
+    for name, config in stores_config.items():
+        dj.config.object_storage.stores[name] = config
+
     yield
-    if og_stores_config is None:
-        del dj.config["stores"]
-    else:
-        dj.config["stores"] = og_stores_config
+
+    # Restore original configuration
+    dj.config.object_storage.project_name = og_project_name
+    dj.config.object_storage.stores.clear()
+    dj.config.object_storage.stores.update(og_stores)
 
 
 @pytest.fixture
@@ -663,31 +686,29 @@ def object_storage_config(tmpdir_factory):
 
 
 @pytest.fixture
-def mock_object_storage(object_storage_config, monkeypatch):
+def mock_object_storage(object_storage_config):
     """Mock object storage configuration in datajoint config."""
-    original_object_storage = getattr(dj.config, "_object_storage", None)
-
-    class MockObjectStorageSettings:
-        def __init__(self, config):
-            self.project_name = config["project_name"]
-            self.protocol = config["protocol"]
-            self.location = config["location"]
-            self.token_length = config.get("token_length", 8)
-            self.partition_pattern = config.get("partition_pattern")
-            self.bucket = config.get("bucket")
-            self.endpoint = config.get("endpoint")
-            self.access_key = config.get("access_key")
-            self.secret_key = config.get("secret_key")
-            self.secure = config.get("secure", True)
-            self.container = config.get("container")
-
-    mock_settings = MockObjectStorageSettings(object_storage_config)
-    monkeypatch.setattr(dj.config, "object_storage", mock_settings)
+    # Save original values
+    original = {
+        "project_name": dj.config.object_storage.project_name,
+        "protocol": dj.config.object_storage.protocol,
+        "location": dj.config.object_storage.location,
+        "token_length": dj.config.object_storage.token_length,
+    }
+
+    # Set test values
+    dj.config.object_storage.project_name = object_storage_config["project_name"]
+    dj.config.object_storage.protocol = object_storage_config["protocol"]
+    dj.config.object_storage.location = object_storage_config["location"]
+    dj.config.object_storage.token_length = object_storage_config.get("token_length", 8)
 
     yield object_storage_config
 
-    if original_object_storage is not None:
-        monkeypatch.setattr(dj.config, "object_storage", original_object_storage)
+    # Restore original values
+    dj.config.object_storage.project_name = original["project_name"]
+    dj.config.object_storage.protocol = original["protocol"]
+    dj.config.object_storage.location = original["location"]
+    dj.config.object_storage.token_length = original["token_length"]
 
 
 @pytest.fixture

tests/schema.py

@@ -200,8 +200,8 @@ class Channel(dj.Part):
         -> master
         channel    :tinyint unsigned   # channel number within Ephys
         ----
-        voltage    : longblob
-        current = null : longblob   # optional current to test null handling
+        voltage    : <djblob>
+        current = null : <djblob>   # optional current to test null handling
         """
 
     def _make_tuples(self, key):
@@ -228,7 +228,7 @@ class Image(dj.Manual):
     # table for testing blob inserts
     id           : int # image identifier
     ---
-    img             : longblob # image
+    img             : <djblob> # image
     """
 
 
@@ -454,7 +454,7 @@ class Longblob(dj.Manual):
     definition = """
     id: int
     ---
-    data: longblob
+    data: <djblob>
     """
 
 

tests/schema_alter.py

@@ -20,7 +20,7 @@ class Experiment(dj.Imported):
     experiment_id  :smallint  # experiment number for this subject
     ---
     data_path     : int  # some number
-    extra=null : longblob  # just testing
+    extra=null : <djblob>  # just testing
     -> [nullable] User
     subject_notes=null         :varchar(2048) # {notes} e.g. purpose of experiment
     entry_time=CURRENT_TIMESTAMP :timestamp   # automatic timestamp