Skip to content

Commit e237515

Browse files
markbaderfm3
andauthored
Enable metadata access for remote datasets. (#1163)
* Enable metadata access for remote datasets. * Update model for remote folder. * Change attr decorators. * Update changelog. * Implement requested feedback. * Add warning when there are duplicates within the keys. * Add entry for RemoteFolder in wk-libs docs. * Implement metadata objects that fetch their data on access. * Implement Metadata class as MutableMapping. * Fix typing. * remove unused imports --------- Co-authored-by: Florian M <[email protected]> Co-authored-by: Florian M <[email protected]>
1 parent a578d30 commit e237515

File tree

10 files changed

+283
-7
lines changed

10 files changed

+283
-7
lines changed

docs/mkdocs.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ nav:
125125
- Layer: api/webknossos/dataset/layer.md
126126
- MagView: api/webknossos/dataset/mag_view.md
127127
- View: api/webknossos/dataset/view.md
128+
- RemoteFolder: api/webknossos/dataset/remote_folder.md
128129
- Annotation: api/webknossos/annotation/annotation.md
129130
- Skeleton:
130131
- Skeleton: api/webknossos/skeleton/skeleton.md

webknossos/Changelog.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ For upgrade instructions, please check the respective _Breaking Changes_ section
1515
### Breaking Changes
1616

1717
### Added
18+
- Enable metadata access for remote datasets. [#1163](https://github.com/scalableminds/webknossos-libs/pull/1163)
1819

1920
### Changed
2021

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import webknossos as wk
2+
3+
4+
def main() -> None:
5+
with wk.webknossos_context(url="https://webknossos.org/"):
6+
l4_sample_dataset = wk.Dataset.open_remote("l4_sample")
7+
# Access the metadata of the dataset
8+
print(l4_sample_dataset.metadata)
9+
10+
# Edit the metadata of the dataset
11+
l4_sample_dataset.metadata["new_key"] = "new_value"
12+
13+
# Access metadata of a folder
14+
print(l4_sample_dataset.folder.metadata)
15+
16+
# Edit the metadata of the folder
17+
l4_sample_dataset.folder.metadata["new_folder_key"] = "new_folder_value"
18+
19+
20+
if __name__ == "__main__":
21+
main()

webknossos/webknossos/client/api_client/_abstract_api_client.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ def _get_json_paginated(
5656
response, response_type
5757
), self._extract_total_count_header(response)
5858

59+
def _put_json(self, route: str, body_structured: Any) -> None:
60+
body_json = self._prepare_for_json(body_structured)
61+
self._put(route, body_json)
62+
5963
def _patch_json(self, route: str, body_structured: Any) -> None:
6064
body_json = self._prepare_for_json(body_structured)
6165
self._patch(route, body_json)
@@ -125,6 +129,27 @@ def _patch(
125129
timeout_seconds=timeout_seconds,
126130
)
127131

132+
def _put(
133+
self,
134+
route: str,
135+
body_json: Optional[Any] = None,
136+
query: Optional[Query] = None,
137+
multipart_data: Optional[httpx._types.RequestData] = None,
138+
files: Optional[httpx._types.RequestFiles] = None,
139+
retry_count: int = 0,
140+
timeout_seconds: Optional[float] = None,
141+
) -> httpx.Response:
142+
return self._request(
143+
"PUT",
144+
route,
145+
body_json=body_json,
146+
multipart_data=multipart_data,
147+
files=files,
148+
query=query,
149+
retry_count=retry_count,
150+
timeout_seconds=timeout_seconds,
151+
)
152+
128153
def _post(
129154
self,
130155
route: str,

webknossos/webknossos/client/api_client/models.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,13 +49,21 @@ class ApiBoundingBox:
4949
depth: int
5050

5151

52+
@attr.s(auto_attribs=True)
53+
class ApiAdditionalAxis:
54+
name: str
55+
bounds: Tuple[int, int]
56+
index: int
57+
58+
5259
@attr.s(auto_attribs=True)
5360
class ApiDataLayer:
5461
name: str
5562
category: str
5663
element_class: str
5764
bounding_box: ApiBoundingBox
5865
resolutions: List[Tuple[int, int, int]]
66+
additional_axes: Optional[List[ApiAdditionalAxis]] = None
5967
largest_segment_id: Optional[int] = None
6068
default_view_configuration: Optional[Dict[str, Any]] = None
6169

@@ -73,6 +81,13 @@ class ApiDataSource:
7381
scale: Optional[ApiVoxelSize] = None
7482

7583

84+
@attr.s(auto_attribs=True)
85+
class ApiMetadata:
86+
key: str
87+
type: str
88+
value: Any
89+
90+
7691
@attr.s(auto_attribs=True)
7792
class ApiDataset:
7893
name: str
@@ -82,6 +97,7 @@ class ApiDataset:
8297
tags: List[str]
8398
data_store: ApiDataStore
8499
data_source: ApiDataSource
100+
metadata: Optional[List[ApiMetadata]] = None
85101
display_name: Optional[str] = None
86102
description: Optional[str] = None
87103

@@ -291,3 +307,13 @@ class ApiFolderWithParent:
291307
id: str
292308
name: str
293309
parent: Optional[str] = None
310+
311+
312+
@attr.s(auto_attribs=True)
313+
class ApiFolder:
314+
id: str
315+
name: str
316+
allowed_teams: List[ApiTeam]
317+
allowed_teams_cumulative: List[ApiTeam]
318+
is_editable: bool
319+
metadata: Optional[List[ApiMetadata]] = None
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
from contextlib import contextmanager
2+
from typing import (
3+
Any,
4+
Dict,
5+
Generator,
6+
Iterator,
7+
List,
8+
MutableMapping,
9+
Sequence,
10+
TypeVar,
11+
Union,
12+
)
13+
14+
from webknossos.client.api_client.models import ApiDataset, ApiFolder, ApiMetadata
15+
from webknossos.utils import infer_metadata_type, parse_metadata_value
16+
17+
_T = TypeVar("_T", bound="Metadata")
18+
19+
20+
class Metadata(MutableMapping):
21+
__slots__ = ()
22+
_api_path: str
23+
_api_type: Any
24+
25+
def __init__(self, _id: str, *args: Any, **kwargs: Dict[str, Any]) -> None:
26+
if not self._api_path or not self._api_type:
27+
raise NotImplementedError(
28+
"This class is not meant to be used directly. Please use FolderMetadata or DatasetMetadata."
29+
)
30+
super().__init__(*args, **kwargs)
31+
self._id: str = _id
32+
self._has_changed: bool = False
33+
self._mapping: Dict = {}
34+
35+
@contextmanager
36+
def _recent_metadata(self: _T) -> Generator[_T, None, None]:
37+
from ..client.context import _get_api_client
38+
39+
try:
40+
client = _get_api_client()
41+
full_object = client._get_json(
42+
f"{self._api_path}{self._id}",
43+
self._api_type, # type: ignore
44+
)
45+
metadata: List[ApiMetadata] = full_object.metadata
46+
if metadata is not None:
47+
self._mapping = {
48+
element.key: parse_metadata_value(element.value, element.type)
49+
for element in metadata
50+
}
51+
else:
52+
self._mapping = {}
53+
yield self
54+
finally:
55+
if self._has_changed:
56+
api_metadata = [
57+
ApiMetadata(key=k, type=infer_metadata_type(v), value=v)
58+
for k, v in self._mapping.items()
59+
]
60+
61+
full_object.metadata = api_metadata
62+
if self._api_type == ApiDataset:
63+
client._patch_json(f"{self._api_path}{self._id}", full_object)
64+
else:
65+
client._put_json(f"{self._api_path}{self._id}", full_object)
66+
self._has_changed = False
67+
68+
def __setitem__(
69+
self, key: str, value: Union[str, int, float, Sequence[str]]
70+
) -> None:
71+
with self._recent_metadata() as metadata:
72+
metadata._has_changed = True
73+
metadata._mapping[key] = value
74+
75+
def __getitem__(self, key: str) -> Union[str, int, float, Sequence[str]]:
76+
with self._recent_metadata() as metadata:
77+
return metadata._mapping[key]
78+
79+
def __delitem__(self, key: str) -> None:
80+
with self._recent_metadata() as metadata:
81+
metadata._has_changed = True
82+
del metadata._mapping[key]
83+
84+
def __contains__(self, key: object) -> bool:
85+
with self._recent_metadata() as metadata:
86+
return key in metadata._mapping
87+
88+
def __eq__(self, other: object) -> bool:
89+
if not isinstance(other, Metadata):
90+
raise NotImplementedError(
91+
f"Cannot compare {self.__class__.__name__} with {other.__class__.__name__}"
92+
)
93+
with self._recent_metadata() as metadata:
94+
return metadata._mapping == other._mapping
95+
96+
def __ne__(self, other: object) -> bool:
97+
return not self == other
98+
99+
def __iter__(self) -> Iterator[Any]:
100+
with self._recent_metadata() as metadata:
101+
return iter(metadata._mapping)
102+
103+
def __len__(self) -> int:
104+
with self._recent_metadata() as metadata:
105+
return len(metadata._mapping)
106+
107+
def __repr__(self) -> str:
108+
with self._recent_metadata() as metadata:
109+
return f"{self.__class__.__name__}({repr(metadata._mapping)})"
110+
111+
112+
class FolderMetadata(Metadata):
113+
_api_path = "/folders/"
114+
_api_type = ApiFolder
115+
116+
117+
class DatasetMetadata(Metadata):
118+
_api_path = "/datasets/"
119+
_api_type = ApiDataset

webknossos/webknossos/dataset/dataset.py

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,10 @@
3535
from numpy.typing import DTypeLike
3636
from upath import UPath
3737

38+
from webknossos.dataset._metadata import DatasetMetadata
3839
from webknossos.geometry.vec_int import VecIntLike
3940

40-
from ..client.api_client.models import ApiDataset
41+
from ..client.api_client.models import ApiDataset, ApiMetadata
4142
from ..geometry.vec3_int import Vec3Int, Vec3IntLike
4243
from ._array import ArrayException, ArrayInfo, BaseArray
4344
from ._utils import pims_images
@@ -69,6 +70,7 @@
6970
copytree,
7071
count_defined_values,
7172
get_executor_for_args,
73+
infer_metadata_type,
7274
is_fs_path,
7375
named_partial,
7476
rmtree,
@@ -2083,6 +2085,7 @@ def _update_dataset_info(
20832085
is_public: bool = _UNSET,
20842086
folder_id: str = _UNSET,
20852087
tags: List[str] = _UNSET,
2088+
metadata: Optional[List[ApiMetadata]] = _UNSET,
20862089
) -> None:
20872090
from ..client.context import _get_api_client
20882091

@@ -2100,14 +2103,32 @@ def _update_dataset_info(
21002103
info.is_public = is_public
21012104
if folder_id is not _UNSET:
21022105
info.folder_id = folder_id
2103-
if display_name is not _UNSET:
2104-
info.display_name = display_name
2106+
if metadata is not _UNSET:
2107+
info.metadata = metadata
21052108

21062109
with self._context:
21072110
_get_api_client().dataset_update(
21082111
self._organization_id, self._dataset_name, info
21092112
)
21102113

2114+
@property
2115+
def metadata(self) -> DatasetMetadata:
2116+
return DatasetMetadata(f"{self._organization_id}/{self._dataset_name}")
2117+
2118+
@metadata.setter
2119+
def metadata(
2120+
self,
2121+
metadata: Optional[
2122+
Union[Dict[str, Union[str, int, float, Sequence[str]]], DatasetMetadata]
2123+
],
2124+
) -> None:
2125+
if metadata is not None:
2126+
api_metadata = [
2127+
ApiMetadata(key=k, type=infer_metadata_type(v), value=v)
2128+
for k, v in metadata.items()
2129+
]
2130+
self._update_dataset_info(metadata=api_metadata)
2131+
21112132
@property
21122133
def display_name(self) -> Optional[str]:
21132134
return self._get_dataset_info().display_name

webknossos/webknossos/dataset/remote_folder.py

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1-
from typing import Iterable, List
1+
from typing import Dict, Iterable, List, Optional, Sequence, Union
22

33
import attr
44

5-
from ..client.api_client.models import ApiFolderWithParent
5+
from webknossos.dataset._metadata import FolderMetadata
6+
from webknossos.utils import infer_metadata_type
7+
8+
from ..client.api_client.models import ApiFolder, ApiFolderWithParent, ApiMetadata
69

710

811
def _get_folder_path(
@@ -15,8 +18,10 @@ def _get_folder_path(
1518
return f"{_get_folder_path(next(f for f in all_folders if f.id == folder.parent), all_folders)}/{folder.name}"
1619

1720

18-
@attr.frozen
21+
@attr.define
1922
class RemoteFolder:
23+
"""This class is used to access and edit metadata of a folder on the webknossos server."""
24+
2025
id: str
2126
name: str
2227

@@ -47,3 +52,23 @@ def get_by_path(cls, path: str) -> "RemoteFolder":
4752
return cls(name=folder_info.name, id=folder_info.id)
4853

4954
raise KeyError(f"Could not find folder {path}.")
55+
56+
@property
57+
def metadata(self) -> FolderMetadata:
58+
return FolderMetadata(self.id)
59+
60+
@metadata.setter
61+
def metadata(
62+
self, metadata: Optional[Dict[str, Union[str, int, float, Sequence[str]]]]
63+
) -> None:
64+
from ..client.context import _get_api_client
65+
66+
client = _get_api_client(enforce_auth=True)
67+
folder = client._get_json(f"/folders/{self.id}", ApiFolder)
68+
if metadata is not None:
69+
api_metadata = [
70+
ApiMetadata(key=k, type=infer_metadata_type(v), value=v)
71+
for k, v in metadata.items()
72+
]
73+
folder.metadata = api_metadata
74+
client._put_json(f"/folders/{self.id}", folder)

webknossos/webknossos/skeleton/skeleton.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
Vector3 = Tuple[float, float, float]
1313

1414

15-
@attr.define()
15+
@attr.define
1616
class Skeleton(Group):
1717
"""
1818
Representation of the [skeleton](/webknossos/skeleton_annotation.html) of an `Annotation`.

0 commit comments

Comments
 (0)