Implement object column type for managed file storage

This commit adds a new `object` column type that provides managed file/folder
storage with fsspec backend integration. Key features:

- Object type declaration in declare.py (stores as JSON in MySQL)
- ObjectRef class for fetch behavior with fsspec accessors (.fs, .store, .full_path)
- Insert processing for file paths, folder paths, and (ext, stream) tuples
- staged_insert1 context manager for direct writes (Zarr/xarray compatibility)
- Path generation with partition pattern support
- Store metadata file (datajoint_store.json) verification/creation
- Folder manifest files for integrity verification

The object type stores metadata inline (no hidden tables), supports multiple
storage backends via fsspec (file, S3, GCS, Azure), and provides ObjectRef
handles on fetch with direct storage access.

Author: claude

src/datajoint/__init__.py

@@ -52,6 +52,7 @@
     "key_hash",
     "logger",
     "cli",
+    "ObjectRef",
 ]
 
 from . import errors
@@ -66,6 +67,7 @@
 from .fetch import key
 from .hash import key_hash
 from .logging import logger
+from .objectref import ObjectRef
 from .schemas import Schema, VirtualModule, list_schemas
 from .settings import config
 from .table import FreeTable, Table

src/datajoint/declare.py

@@ -64,6 +64,7 @@
         INTERNAL_ATTACH=r"attach$",
         EXTERNAL_ATTACH=r"attach@(?P<store>[a-z][\-\w]*)$",
         FILEPATH=r"filepath@(?P<store>[a-z][\-\w]*)$",
+        OBJECT=r"object$",  # managed object storage (files/folders)
         UUID=r"uuid$",
         ADAPTED=r"<.+>$",
     ).items()
@@ -76,6 +77,7 @@
     "EXTERNAL_ATTACH",
     "EXTERNAL_BLOB",
     "FILEPATH",
+    "OBJECT",
     "ADAPTED",
 } | set(TYPE_ALIASES)
 NATIVE_TYPES = set(TYPE_PATTERN) - SPECIAL_TYPES
@@ -464,6 +466,9 @@ def substitute_special_type(match, category, foreign_key_sql, context):
         match["type"] = UUID_DATA_TYPE
     elif category == "INTERNAL_ATTACH":
         match["type"] = "LONGBLOB"
+    elif category == "OBJECT":
+        # Object type stores metadata as JSON - no foreign key to external table
+        match["type"] = "JSON"
     elif category in EXTERNAL_TYPES:
         if category == "FILEPATH" and not _support_filepath_types():
             raise DataJointError(

src/datajoint/fetch.py

@@ -12,7 +12,9 @@
 
 from . import blob, hash
 from .errors import DataJointError
+from .objectref import ObjectRef
 from .settings import config
+from .storage import StorageBackend
 from .utils import safe_write
 
 
@@ -48,6 +50,15 @@ def _get(connection, attr, data, squeeze, download_path):
     """
     if data is None:
         return
+    if attr.is_object:
+        # Object type - return ObjectRef handle
+        json_data = json.loads(data) if isinstance(data, str) else data
+        try:
+            spec = config.get_object_storage_spec()
+            backend = StorageBackend(spec)
+        except DataJointError:
+            backend = None
+        return ObjectRef.from_json(json_data, backend=backend)
     if attr.json:
         return json.loads(data)
 

src/datajoint/heading.py

@@ -32,6 +32,7 @@
     is_blob=False,
     is_attachment=False,
     is_filepath=False,
+    is_object=False,
     is_external=False,
     is_hidden=False,
     adapter=None,
@@ -136,7 +137,7 @@ def blobs(self):
 
     @property
     def non_blobs(self):
-        return [k for k, v in self.attributes.items() if not (v.is_blob or v.is_attachment or v.is_filepath or v.json)]
+        return [k for k, v in self.attributes.items() if not (v.is_blob or v.is_attachment or v.is_filepath or v.is_object or v.json)]
 
     @property
     def new_attributes(self):
@@ -262,6 +263,7 @@ def _init_from_database(self):
                 json=bool(TYPE_PATTERN["JSON"].match(attr["type"])),
                 is_attachment=False,
                 is_filepath=False,
+                is_object=False,
                 adapter=None,
                 store=None,
                 is_external=False,
@@ -325,6 +327,7 @@ def _init_from_database(self):
                     unsupported=False,
                     is_attachment=category in ("INTERNAL_ATTACH", "EXTERNAL_ATTACH"),
                     is_filepath=category == "FILEPATH",
+                    is_object=category == "OBJECT",
                     # INTERNAL_BLOB is not a custom type but is included for completeness
                     is_blob=category in ("INTERNAL_BLOB", "EXTERNAL_BLOB"),
                     uuid=category == "UUID",
@@ -337,10 +340,11 @@ def _init_from_database(self):
                     attr["is_blob"],
                     attr["is_attachment"],
                     attr["is_filepath"],
+                    attr["is_object"],
                     attr["json"],
                 )
             ):
-                raise DataJointError("Json, Blob, attachment, or filepath attributes are not allowed in the primary key")
+                raise DataJointError("Json, Blob, attachment, filepath, or object attributes are not allowed in the primary key")
 
             if attr["string"] and attr["default"] is not None and attr["default"] not in sql_literals:
                 attr["default"] = '"%s"' % attr["default"]