Skip to content

Commit cc77b37

Browse files
feat: enforce reserved sections for filepath codec
- Add validation to prevent filepath paths starting with _hash/ or _schema/ - Update FilepathCodec docstring to clarify reserved sections - Filepath gives users maximum freedom while protecting DataJoint-managed sections - Users can organize files anywhere in store except reserved sections
1 parent 1366776 commit cc77b37

File tree

1 file changed

+29
-3
lines changed

1 file changed

+29
-3
lines changed

src/datajoint/builtin_codecs.py

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -733,10 +733,16 @@ class FilepathCodec(Codec):
733733
734734
External only - requires @store.
735735
736+
This codec gives users maximum freedom in organizing their files while
737+
reusing DataJoint's store configuration. Files can be placed anywhere
738+
in the store EXCEPT the reserved ``_hash/`` and ``_schema/`` sections
739+
which are managed by DataJoint.
740+
736741
This is useful when:
737742
- Files are managed externally (e.g., by acquisition software)
738743
- Files are too large to copy
739744
- You want to reference shared datasets
745+
- You need custom directory structures
740746
741747
Example::
742748
@@ -749,6 +755,7 @@ class Recordings(dj.Manual):
749755
'''
750756
751757
# Reference an existing file (no copy)
758+
# Path is relative to store location
752759
table.insert1({'recording_id': 1, 'raw_data': 'subject01/session001/data.bin'})
753760
754761
# Fetch returns ObjectRef for lazy access
@@ -757,7 +764,10 @@ class Recordings(dj.Manual):
757764
ref.download() # Download to local path
758765
759766
Storage Format:
760-
JSON metadata: ``{path, store}``
767+
JSON metadata: ``{path, store, size, timestamp}``
768+
769+
Reserved Sections:
770+
Paths cannot start with ``_hash/`` or ``_schema/`` - these are managed by DataJoint.
761771
762772
Warning:
763773
The file must exist in the store at the specified path.
@@ -779,7 +789,7 @@ def encode(self, value: Any, *, key: dict | None = None, store_name: str | None
779789
Parameters
780790
----------
781791
value : str
782-
Relative path within the store.
792+
Relative path within the store. Cannot use reserved sections (_hash/, _schema/).
783793
key : dict, optional
784794
Primary key values (unused).
785795
store_name : str, optional
@@ -789,14 +799,30 @@ def encode(self, value: Any, *, key: dict | None = None, store_name: str | None
789799
-------
790800
dict
791801
Metadata dict: ``{path, store}``.
802+
803+
Raises
804+
------
805+
ValueError
806+
If path uses reserved sections (_hash/ or _schema/).
807+
FileNotFoundError
808+
If file does not exist in the store.
792809
"""
793810
from datetime import datetime, timezone
794811

795812
from .hash_registry import get_store_backend
796813

797814
path = str(value)
798815

799-
# Optionally verify file exists
816+
# Validate path doesn't use reserved sections
817+
path_normalized = path.lstrip('/')
818+
if path_normalized.startswith('_hash/') or path_normalized.startswith('_schema/'):
819+
raise ValueError(
820+
f"<filepath@> cannot use reserved sections '_hash/' or '_schema/'. "
821+
f"These sections are managed by DataJoint. "
822+
f"Got path: {path}"
823+
)
824+
825+
# Verify file exists
800826
backend = get_store_backend(store_name)
801827
if not backend.exists(path):
802828
raise FileNotFoundError(f"File not found in store '{store_name or 'default'}': {path}")

0 commit comments

Comments
 (0)