Merge pull request #1333 from datajoint/feature/unified-stores-config

Unified stores configuration with configurable prefixes and filepath_default

Author: dimitri-yatsenko

integration_test_summary.txt

@@ -0,0 +1,33 @@
+## Integration Test Results - FINAL
+
+**Test Summary:**
+- ✅ 520 tests PASSED
+- ⏭️  7 tests SKIPPED
+- ❌ 0 tests FAILED
+
+**All tests passing!**
+
+### Initial Issues Found and Fixed
+
+Initial run had 24 failures in object storage tests due to test fixture bug:
+- `conftest.py`: object_storage_config wasn't creating the `test_project` subdirectory
+- `test_update1.py`: mock_stores_update wasn't creating `djtest` subdirectories
+
+**Root cause:** Test fixtures were configuring storage locations but not creating
+the directories. StorageBackend validates that file protocol locations exist
+during initialization.
+
+**Fix:** Added `Path(location).mkdir(parents=True, exist_ok=True)` in both fixtures.
+
+### Test Coverage Verified
+
+All unified stores configuration functionality tested:
+- ✅ Configuration system with stores.default and stores.filepath_default
+- ✅ Prefix validation and separation (hash_prefix, schema_prefix, filepath_prefix)
+- ✅ Filepath codec validation with dynamic prefix checking
+- ✅ Store backend initialization and validation
+- ✅ Object storage (file, stream, folder operations)
+- ✅ Hash-addressed storage (blob, attach)
+- ✅ Schema-addressed storage (object, npy)
+- ✅ All relational operators and queries
+- ✅ Schema management and dependencies

src/datajoint/builtin_codecs.py

@@ -323,13 +323,16 @@ def _build_path(
         field: str,
         primary_key: dict,
         ext: str | None = None,
+        store_name: str | None = None,
     ) -> tuple[str, str]:
         """
         Build schema-addressed storage path.
 
         Constructs a path that mirrors the database schema structure:
         ``{schema}/{table}/{pk_values}/{field}{ext}``
 
+        Supports partitioning if configured in the store.
+
         Parameters
         ----------
         schema : str
@@ -342,6 +345,8 @@ def _build_path(
             Primary key values.
         ext : str, optional
             File extension (e.g., ".npy", ".zarr").
+        store_name : str, optional
+            Store name for retrieving partition configuration.
 
         Returns
         -------
@@ -350,13 +355,21 @@ def _build_path(
             is a unique identifier.
         """
         from .storage import build_object_path
+        from . import config
+
+        # Get store configuration for partition_pattern and token_length
+        spec = config.get_store_spec(store_name)
+        partition_pattern = spec.get("partition_pattern")
+        token_length = spec.get("token_length", 8)
 
         return build_object_path(
             schema=schema,
             table=table,
             field=field,
             primary_key=primary_key,
             ext=ext,
+            partition_pattern=partition_pattern,
+            token_length=token_length,
         )
 
     def _get_backend(self, store_name: str | None = None):
@@ -518,7 +531,7 @@ def encode(
             raise TypeError(f"<object> expects bytes or path, got {type(value).__name__}")
 
         # Build storage path using inherited helper
-        path, token = self._build_path(schema, table, field, primary_key, ext=ext)
+        path, token = self._build_path(schema, table, field, primary_key, ext=ext, store_name=store_name)
 
         # Get storage backend using inherited helper
         backend = self._get_backend(store_name)
@@ -733,10 +746,16 @@ class FilepathCodec(Codec):
 
     External only - requires @store.
 
+    This codec gives users maximum freedom in organizing their files while
+    reusing DataJoint's store configuration. Files can be placed anywhere
+    in the store EXCEPT the reserved ``_hash/`` and ``_schema/`` sections
+    which are managed by DataJoint.
+
     This is useful when:
     - Files are managed externally (e.g., by acquisition software)
     - Files are too large to copy
     - You want to reference shared datasets
+    - You need custom directory structures
 
     Example::
 
@@ -749,6 +768,7 @@ class Recordings(dj.Manual):
             '''
 
         # Reference an existing file (no copy)
+        # Path is relative to store location
         table.insert1({'recording_id': 1, 'raw_data': 'subject01/session001/data.bin'})
 
         # Fetch returns ObjectRef for lazy access
@@ -757,7 +777,10 @@ class Recordings(dj.Manual):
         ref.download()  # Download to local path
 
     Storage Format:
-        JSON metadata: ``{path, store}``
+        JSON metadata: ``{path, store, size, timestamp}``
+
+    Reserved Sections:
+        Paths cannot start with ``_hash/`` or ``_schema/`` - these are managed by DataJoint.
 
     Warning:
         The file must exist in the store at the specified path.
@@ -769,7 +792,9 @@ class Recordings(dj.Manual):
     def get_dtype(self, is_store: bool) -> str:
         """Filepath is external only."""
         if not is_store:
-            raise DataJointError("<filepath> requires @store")
+            raise DataJointError(
+                "<filepath> requires @ symbol. Use <filepath@> for default store " "or <filepath@store> to specify store."
+            )
         return "json"
 
     def encode(self, value: Any, *, key: dict | None = None, store_name: str | None = None) -> dict:
@@ -779,7 +804,7 @@ def encode(self, value: Any, *, key: dict | None = None, store_name: str | None
         Parameters
         ----------
         value : str
-            Relative path within the store.
+            Relative path within the store. Cannot use reserved sections (_hash/, _schema/).
         key : dict, optional
             Primary key values (unused).
         store_name : str, optional
@@ -789,14 +814,55 @@ def encode(self, value: Any, *, key: dict | None = None, store_name: str | None
         -------
         dict
             Metadata dict: ``{path, store}``.
+
+        Raises
+        ------
+        ValueError
+            If path uses reserved sections (_hash/ or _schema/).
+        FileNotFoundError
+            If file does not exist in the store.
         """
         from datetime import datetime, timezone
 
+        from . import config
         from .hash_registry import get_store_backend
 
         path = str(value)
 
-        # Optionally verify file exists
+        # Get store spec to check prefix configuration
+        # Use filepath_default if no store specified (filepath is not part of OAS)
+        spec = config.get_store_spec(store_name, use_filepath_default=True)
+
+        # Validate path doesn't use reserved sections (hash and schema)
+        path_normalized = path.lstrip("/")
+        reserved_prefixes = []
+
+        hash_prefix = spec.get("hash_prefix")
+        if hash_prefix:
+            reserved_prefixes.append(("hash_prefix", hash_prefix))
+
+        schema_prefix = spec.get("schema_prefix")
+        if schema_prefix:
+            reserved_prefixes.append(("schema_prefix", schema_prefix))
+
+        # Check if path starts with any reserved prefix
+        for prefix_name, prefix_value in reserved_prefixes:
+            prefix_normalized = prefix_value.strip("/") + "/"
+            if path_normalized.startswith(prefix_normalized):
+                raise ValueError(
+                    f"<filepath@> cannot use reserved section '{prefix_value}' ({prefix_name}). "
+                    f"This section is managed by DataJoint. "
+                    f"Got path: {path}"
+                )
+
+        # If filepath_prefix is configured, enforce it
+        filepath_prefix = spec.get("filepath_prefix")
+        if filepath_prefix:
+            filepath_prefix_normalized = filepath_prefix.strip("/") + "/"
+            if not path_normalized.startswith(filepath_prefix_normalized):
+                raise ValueError(f"<filepath@> must use prefix '{filepath_prefix}' (filepath_prefix). " f"Got path: {path}")
+
+        # Verify file exists
         backend = get_store_backend(store_name)
         if not backend.exists(path):
             raise FileNotFoundError(f"File not found in store '{store_name or 'default'}': {path}")
@@ -1179,7 +1245,7 @@ def encode(
         schema, table, field, primary_key = self._extract_context(key)
 
         # Build schema-addressed storage path
-        path, _ = self._build_path(schema, table, field, primary_key, ext=".npy")
+        path, _ = self._build_path(schema, table, field, primary_key, ext=".npy", store_name=store_name)
 
         # Serialize to .npy format
         buffer = io.BytesIO()

src/datajoint/hash_registry.py

@@ -138,20 +138,15 @@ def get_store_backend(store_name: str | None = None) -> StorageBackend:
     Parameters
     ----------
     store_name : str, optional
-        Name of the store to use. If None, uses the default object storage
-        configuration or the configured default_store.
+        Name of the store to use. If None, uses stores.default.
 
     Returns
     -------
     StorageBackend
         StorageBackend instance.
     """
-    # If store_name is None, check for configured default_store
-    if store_name is None and config.object_storage.default_store:
-        store_name = config.object_storage.default_store
-
-    # get_object_store_spec handles None by returning default object_storage config
-    spec = config.get_object_store_spec(store_name)
+    # get_store_spec handles None by using stores.default
+    spec = config.get_store_spec(store_name)
     return StorageBackend(spec)
 
 
@@ -162,14 +157,14 @@ def get_store_subfolding(store_name: str | None = None) -> tuple[int, ...] | Non
     Parameters
     ----------
     store_name : str, optional
-        Name of the store. If None, uses default store.
+        Name of the store. If None, uses stores.default.
 
     Returns
     -------
     tuple[int, ...] | None
         Subfolding pattern (e.g., (2, 2)) or None for flat storage.
     """
-    spec = config.get_object_store_spec(store_name)
+    spec = config.get_store_spec(store_name)
     subfolding = spec.get("subfolding")
     if subfolding is not None:
         return tuple(subfolding)