Skip to content

Commit c3e0163

Browse files
Support partition_pattern in schema-addressed storage
- Updated SchemaCodec._build_path() to accept store_name parameter - _build_path() now retrieves partition_pattern and token_length from store spec - ObjectCodec and NpyCodec encode methods pass store_name to _build_path - Enables partitioning configuration like partition_pattern: '{mouse_id}/{session_date}' This allows organizing storage by experimental structure: - Without: {schema}/{table}/{mouse_id=X}/{session_date=Y}/... - With: {mouse_id=X}/{session_date=Y}/{schema}/{table}/... Partitioning makes storage browsable by subject/session and enables selective sync/backup of individual subjects or sessions.
1 parent 5512659 commit c3e0163

File tree

1 file changed

+19
-2
lines changed

1 file changed

+19
-2
lines changed

src/datajoint/builtin_codecs.py

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -323,13 +323,16 @@ def _build_path(
323323
field: str,
324324
primary_key: dict,
325325
ext: str | None = None,
326+
store_name: str | None = None,
326327
) -> tuple[str, str]:
327328
"""
328329
Build schema-addressed storage path.
329330
330331
Constructs a path that mirrors the database schema structure:
331332
``{schema}/{table}/{pk_values}/{field}{ext}``
332333
334+
Supports partitioning if configured in the store.
335+
333336
Parameters
334337
----------
335338
schema : str
@@ -342,6 +345,8 @@ def _build_path(
342345
Primary key values.
343346
ext : str, optional
344347
File extension (e.g., ".npy", ".zarr").
348+
store_name : str, optional
349+
Store name for retrieving partition configuration.
345350
346351
Returns
347352
-------
@@ -350,13 +355,21 @@ def _build_path(
350355
is a unique identifier.
351356
"""
352357
from .storage import build_object_path
358+
from . import config
359+
360+
# Get store configuration for partition_pattern and token_length
361+
spec = config.get_store_spec(store_name)
362+
partition_pattern = spec.get("partition_pattern")
363+
token_length = spec.get("token_length", 8)
353364

354365
return build_object_path(
355366
schema=schema,
356367
table=table,
357368
field=field,
358369
primary_key=primary_key,
359370
ext=ext,
371+
partition_pattern=partition_pattern,
372+
token_length=token_length,
360373
)
361374

362375
def _get_backend(self, store_name: str | None = None):
@@ -518,7 +531,9 @@ def encode(
518531
raise TypeError(f"<object> expects bytes or path, got {type(value).__name__}")
519532

520533
# Build storage path using inherited helper
521-
path, token = self._build_path(schema, table, field, primary_key, ext=ext)
534+
path, token = self._build_path(
535+
schema, table, field, primary_key, ext=ext, store_name=store_name
536+
)
522537

523538
# Get storage backend using inherited helper
524539
backend = self._get_backend(store_name)
@@ -1232,7 +1247,9 @@ def encode(
12321247
schema, table, field, primary_key = self._extract_context(key)
12331248

12341249
# Build schema-addressed storage path
1235-
path, _ = self._build_path(schema, table, field, primary_key, ext=".npy")
1250+
path, _ = self._build_path(
1251+
schema, table, field, primary_key, ext=".npy", store_name=store_name
1252+
)
12361253

12371254
# Serialize to .npy format
12381255
buffer = io.BytesIO()

0 commit comments

Comments
 (0)