Add 'properties' to DataSources (#1268)

* MNT: Update readme

* ENH: add properties column to the DataSources table

* MNT: Update changelog

* DOC: update documentation

* TST: test DataSource.properties

Author: genematx

CHANGELOG.md

@@ -21,10 +21,9 @@ Write the date in place of the "Unreleased" in the case a new version is release
 ### Fixed
 
 - Error handling in `tiled.client.download._download_url`.
-
-### Fixed
-
 - Slow performance when deleting nodes in large catalogs with many nodes.
+- Add `properties` field to the DataSource object and table, to store additional
+  metadata about the data source (e.g. array chunking information).
 
 ## v0.2.3 (2025-12-17)
 

docs/source/explanations/catalog.md

@@ -53,6 +53,9 @@ and `assets`, describes the format,  structure, and location of the data.
   (This is used by Tiled to identify a suitable Adapter to read this data.)
 - `parameters` --- JSON object with additional parameters that will be passed
   to the Adapter
+- `properties` --- JSON object with additional properties of the native data format,
+  such as compression type, chunking, column names, etc. These are not passed to the Adapter,
+  but are stored for informational purposes.
 - `management` --- enum indicating whether the data is registered `"external"` data
   or `"writable"` data managed by Tiled
 - `structure_family` --- enum of structure types (`"container"`, `"array"`, `"table"`, ...)

tests/test_catalog.py

@@ -226,6 +226,7 @@ async def test_write_array_external(a, tmpdir):
                 mimetype="image/tiff",
                 structure=structure,
                 parameters={},
+                properties={"chunks": structure["chunks"]},  # Optional property
                 management="external",
                 assets=[
                     Asset(
@@ -240,6 +241,7 @@ async def test_write_array_external(a, tmpdir):
     )
     x = await a.lookup_adapter(["x"])
     assert numpy.array_equal(await x.read(), arr)
+    assert x.data_sources[0].properties == {"chunks": [[5], [3]]}
 
 
 @pytest.mark.asyncio

tiled/authn_database/migrations/README

@@ -1 +1,7 @@
 Generic single-database configuration.
+
+To generate a new revision file, run:
+
+```
+python -m tiled.authn_database revision -m "description..."
+```

tiled/catalog/adapter.py

@@ -751,6 +751,7 @@ async def create_node(
                     mimetype=data_source.mimetype,
                     management=data_source.management,
                     parameters=data_source.parameters,
+                    properties=data_source.properties,
                     structure_id=structure_id,
                 )
                 db.add(data_source_orm)

tiled/catalog/core.py

@@ -6,6 +6,7 @@
 
 # This is list of all valid revisions (from current to oldest).
 ALL_REVISIONS = [
+    "4cf6011a8db5",
     "a86a48befff6",
     "dfbb7478c6bd",
     "a963a6c32a0c",

tiled/catalog/migrations/README

@@ -1 +1,7 @@
 Generic single-database configuration.
+
+To generate a new revision file, run:
+
+```
+python -m tiled.catalog revision -m "description..."
+```

tiled/catalog/migrations/versions/4cf6011a8db5_add_properties_to_datasources.py

@@ -0,0 +1,28 @@
+"""Add 'properties' column to the DataSources table
+
+Revision ID: 4cf6011a8db5
+Revises: a86a48befff6
+Create Date: 2026-01-09 13:59:48.448079
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+from tiled.catalog.orm import JSONVariant
+
+# revision identifiers, used by Alembic.
+revision = "4cf6011a8db5"
+down_revision = "a86a48befff6"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "data_sources",
+        sa.Column("properties", JSONVariant, nullable=False, server_default="{}"),
+    )
+
+
+def downgrade():
+    op.drop_column("data_sources", "properties")

tiled/catalog/orm.py

@@ -467,6 +467,8 @@ class DataSource(Timestamped, Base):
     # These are additional parameters passed to the Adapter to guide
     # it to access and arrange the data in the file correctly.
     parameters = Column(JSONVariant, nullable=True)
+    # These are additional properties describing the data source (e.g. array chunks).
+    properties = Column(JSONVariant, nullable=True)
     # This relates to the mutability of the data.
     management = Column(Enum(Management), nullable=False)
     structure_family = Column(Enum(StructureFamily), nullable=False)

tiled/server/schemas.py

@@ -149,6 +149,7 @@ class DataSource(pydantic.BaseModel, Generic[StructureT]):
     structure: Optional[StructureT]
     mimetype: Optional[str] = None
     parameters: dict = {}
+    properties: dict = {}
     assets: List[Asset] = []
     management: Management = Management.writable
 
@@ -167,6 +168,7 @@ def from_orm(cls, orm: tiled.catalog.orm.DataSource) -> DataSource:
             structure=structure,
             mimetype=orm.mimetype,
             parameters=orm.parameters,
+            properties=orm.properties,
             assets=[Asset.from_assoc_orm(assoc) for assoc in orm.asset_associations],
             management=orm.management,
         )