Skip to content

Commit 739ec47

Browse files
olevskiolevski
authored
feat(cli): set list of custom metadata for project and dataset (#3165)
1 parent 9dd6a2d commit 739ec47

File tree

13 files changed

+320
-62
lines changed

13 files changed

+320
-62
lines changed

‎renku/core/dataset/dataset.py

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,8 @@ def edit_dataset(
194194
creators: Optional[Union[List[Person], NoValueType]],
195195
keywords: Optional[Union[List[str], NoValueType]] = NO_VALUE,
196196
images: Optional[Union[List[ImageRequestModel], NoValueType]] = NO_VALUE,
197-
custom_metadata: Optional[Union[Dict, NoValueType]] = NO_VALUE,
197+
custom_metadata: Optional[Union[Dict, List[Dict], NoValueType]] = NO_VALUE,
198+
custom_metadata_source: Optional[Union[str, NoValueType]] = NO_VALUE,
198199
):
199200
"""Edit dataset metadata.
200201
@@ -206,7 +207,10 @@ def edit_dataset(
206207
keywords(Optional[Union[List[str], NoValueType]]): New keywords for dataset (Default value = ``NO_VALUE``).
207208
images(Optional[Union[List[ImageRequestModel], NoValueType]]): New images for dataset
208209
(Default value = ``NO_VALUE``).
209-
custom_metadata(Optional[Union[Dict, NoValueType]]): Custom JSON-LD metadata (Default value = ``NO_VALUE``).
210+
custom_metadata(Optional[Union[Dict, List[Dict], NoValueType]]): Custom JSON-LD metadata
211+
(Default value = ``NO_VALUE``).
212+
custom_metadata_source(Optional[Union[str, NoValueType]]): The custom metadata source
213+
(Default value = ``NO_VALUE``).
210214
211215
Returns:
212216
bool: True if updates were performed.
@@ -246,7 +250,11 @@ def edit_dataset(
246250
)
247251

248252
if custom_metadata is not NO_VALUE:
249-
update_dataset_custom_metadata(dataset, cast(Optional[Dict], custom_metadata))
253+
update_dataset_custom_metadata(
254+
dataset,
255+
cast(Optional[Union[Dict, List[Dict]]], custom_metadata),
256+
cast(Optional[str], custom_metadata_source),
257+
)
250258
updated["custom_metadata"] = custom_metadata
251259

252260
if not updated:
@@ -844,18 +852,28 @@ def set_dataset_images(dataset: Dataset, images: Optional[List[ImageRequestModel
844852
return images_updated or dataset.images != previous_images
845853

846854

847-
def update_dataset_custom_metadata(dataset: Dataset, custom_metadata: Optional[Dict]):
855+
def update_dataset_custom_metadata(
856+
dataset: Dataset,
857+
custom_metadata: Optional[Union[Dict, List[Dict]]],
858+
custom_metadata_source: Optional[str],
859+
):
848860
"""Update custom metadata on a dataset.
849861
850862
Args:
851863
dataset(Dataset): The dataset to update.
852864
custom_metadata(Dict): Custom JSON-LD metadata to set.
865+
custom_metadata_source(str): The source field for the custom metadata.
853866
"""
854867

855-
existing_metadata = [a for a in dataset.annotations if a.source != "renku"]
868+
existing_metadata = [a for a in dataset.annotations if a.source != custom_metadata_source]
856869

857-
if custom_metadata is not None:
858-
existing_metadata.append(Annotation(id=Annotation.generate_id(), body=custom_metadata, source="renku"))
870+
if custom_metadata is not None and custom_metadata_source is not None:
871+
if isinstance(custom_metadata, dict):
872+
custom_metadata = [custom_metadata]
873+
for icustom_metadata in custom_metadata:
874+
existing_metadata.append(
875+
Annotation(id=Annotation.generate_id(), body=icustom_metadata, source=custom_metadata_source)
876+
)
859877

860878
dataset.annotations = existing_metadata
861879

‎renku/core/project.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ def edit_project(
3434
description: Optional[Union[str, NoValueType]],
3535
creator: Union[Dict, str, NoValueType],
3636
keywords: Optional[Union[List[str], NoValueType]],
37-
custom_metadata: Optional[Union[Dict, NoValueType]],
37+
custom_metadata: Optional[Union[Dict, List[Dict], NoValueType]],
38+
custom_metadata_source: str,
3839
project_gateway: IProjectGateway,
3940
):
4041
"""Edit dataset metadata.
@@ -43,7 +44,8 @@ def edit_project(
4344
description(Union[Optional[str], NoValueType]): New description.
4445
creator(Union[Dict, str, NoValueType]): New creators.
4546
keywords(Union[Optional[List[str]]): New keywords.
46-
custom_metadata(Union[Optional[Dict]): Custom JSON-LD metadata.
47+
custom_metadata(Union[Optional[Dict, List[Dict]]): Custom JSON-LD metadata.
48+
custom_metadata_source(Optional[str]): Custom metadata source.
4749
project_gateway(IProjectGateway): Injected project gateway.
4850
4951
Returns:
@@ -67,7 +69,11 @@ def edit_project(
6769
if updated:
6870
project = project_gateway.get_project()
6971
project.update_metadata(
70-
creator=parsed_creator, description=description, keywords=keywords, custom_metadata=custom_metadata
72+
creator=parsed_creator,
73+
description=description,
74+
keywords=keywords,
75+
custom_metadata=custom_metadata,
76+
custom_metadata_source=custom_metadata_source,
7177
)
7278
project_gateway.update_project(project)
7379

‎renku/domain_model/project.py

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ def generate_id(namespace: str, name: str):
176176

177177
return f"/projects/{namespace}/{slug}"
178178

179-
def update_metadata(self, custom_metadata=None, **kwargs):
179+
def update_metadata(self, custom_metadata=None, custom_metadata_source=None, **kwargs):
180180
"""Updates metadata."""
181181
editable_attributes = ["creator", "description", "keywords"]
182182
for name, value in kwargs.items():
@@ -185,10 +185,18 @@ def update_metadata(self, custom_metadata=None, **kwargs):
185185
if value is not NO_VALUE and value != getattr(self, name):
186186
setattr(self, name, value)
187187

188-
if custom_metadata is not NO_VALUE:
189-
existing_metadata = [a for a in self.annotations if a.source != "renku"]
190-
188+
if custom_metadata is not NO_VALUE and custom_metadata_source is not NO_VALUE:
189+
existing_metadata = [a for a in self.annotations if a.source != custom_metadata_source]
191190
if custom_metadata is not None:
192-
existing_metadata.append(Annotation(id=Annotation.generate_id(), body=custom_metadata, source="renku"))
191+
if not isinstance(custom_metadata, list):
192+
custom_metadata = [custom_metadata]
193+
for icustom_metadata in custom_metadata:
194+
existing_metadata.append(
195+
Annotation(
196+
id=Annotation.generate_id(),
197+
body=icustom_metadata,
198+
source=custom_metadata_source,
199+
)
200+
)
193201

194202
self.annotations = existing_metadata

‎renku/ui/cli/dataset.py

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -660,6 +660,12 @@ def create(name, title, description, creators, metadata, keyword, storage, datad
660660
type=click.UNPROCESSED,
661661
help="Custom metadata to be associated with the dataset.",
662662
)
663+
@click.option(
664+
"--metadata-source",
665+
default=NO_VALUE,
666+
type=click.UNPROCESSED,
667+
help="Set the source field in the metadata when editing it if not provided, then the default is 'renku'.",
668+
)
663669
@click.option(
664670
"-k",
665671
"--keyword",
@@ -677,7 +683,7 @@ def create(name, title, description, creators, metadata, keyword, storage, datad
677683
type=click.Choice(["keywords", "k", "images", "i", "metadata", "m"]),
678684
help="Remove keywords from dataset.",
679685
)
680-
def edit(name, title, description, creators, metadata, keywords, unset):
686+
def edit(name, title, description, creators, metadata, metadata_source, keywords, unset):
681687
"""Edit dataset metadata."""
682688
from renku.command.dataset import edit_dataset_command
683689
from renku.core.util.metadata import construct_creators
@@ -704,6 +710,12 @@ def edit(name, title, description, creators, metadata, keywords, unset):
704710
if "i" in unset or "images" in unset:
705711
images = None
706712

713+
if metadata_source is not NO_VALUE and metadata is NO_VALUE:
714+
raise click.UsageError("The '--metadata-source' option can only be used with the '--metadata' flag")
715+
716+
if metadata_source is NO_VALUE and metadata is not NO_VALUE:
717+
metadata_source = "renku"
718+
707719
custom_metadata = metadata
708720
no_email_warnings = False
709721

@@ -728,6 +740,7 @@ def edit(name, title, description, creators, metadata, keywords, unset):
728740
keywords=keywords,
729741
images=images,
730742
custom_metadata=custom_metadata,
743+
custom_metadata_source=metadata_source,
731744
)
732745
).output
733746

‎renku/ui/cli/project.py

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,13 @@ def project():
7575
type=click.Choice(["keywords", "k", "metadata", "m"]),
7676
help="Remove keywords from dataset.",
7777
)
78-
def edit(description, keywords, creators, metadata, unset):
78+
@click.option(
79+
"--metadata-source",
80+
type=click.UNPROCESSED,
81+
default=NO_VALUE,
82+
help="Set the source field in the metadata when editing it if not provided, then the default is 'renku'.",
83+
)
84+
def edit(description, keywords, creators, metadata, unset, metadata_source):
7985
"""Edit project metadata."""
8086
from renku.command.project import edit_project_command
8187

@@ -95,6 +101,12 @@ def edit(description, keywords, creators, metadata, unset):
95101
raise click.UsageError("Cant use '--metadata' together with unsetting metadata")
96102
metadata = None
97103

104+
if metadata_source is not NO_VALUE and metadata is NO_VALUE:
105+
raise click.UsageError("The '--metadata-source' option can only be used with the '--metadata' flag")
106+
107+
if metadata_source is NO_VALUE and metadata is not NO_VALUE:
108+
metadata_source = "renku"
109+
98110
custom_metadata = metadata
99111

100112
if metadata and metadata is not NO_VALUE:
@@ -107,7 +119,13 @@ def edit(description, keywords, creators, metadata, unset):
107119
result = (
108120
edit_project_command()
109121
.build()
110-
.execute(description=description, creator=creators, keywords=keywords, custom_metadata=custom_metadata)
122+
.execute(
123+
description=description,
124+
creator=creators,
125+
keywords=keywords,
126+
custom_metadata=custom_metadata,
127+
custom_metadata_source=metadata_source,
128+
)
111129
)
112130

113131
updated, no_email_warning = result.output

‎renku/ui/service/controllers/datasets_edit.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,14 @@ def renku_op(self):
9797
else:
9898
custom_metadata = NO_VALUE
9999

100+
if "custom_metadata_source" in self.ctx:
101+
custom_metadata_source = self.ctx.get("custom_metadata")
102+
else:
103+
custom_metadata_source = NO_VALUE
104+
105+
if custom_metadata_source is NO_VALUE and custom_metadata is not NO_VALUE:
106+
custom_metadata_source = "renku"
107+
100108
result = (
101109
edit_dataset_command()
102110
.with_commit_message(self.ctx["commit_message"])
@@ -109,6 +117,7 @@ def renku_op(self):
109117
keywords=keywords,
110118
images=images,
111119
custom_metadata=custom_metadata,
120+
custom_metadata_source=custom_metadata_source,
112121
)
113122
)
114123

‎renku/ui/service/controllers/project_edit.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,14 @@ def renku_op(self):
6464
else:
6565
custom_metadata = NO_VALUE
6666

67+
if "custom_metadata_source" in self.ctx:
68+
custom_metadata_source = self.ctx.get("custom_metadata_source")
69+
else:
70+
custom_metadata_source = NO_VALUE
71+
72+
if custom_metadata_source is NO_VALUE and custom_metadata is not NO_VALUE:
73+
custom_metadata_source = "renku"
74+
6775
if "keywords" in self.ctx:
6876
keywords = self.ctx.get("keywords")
6977
else:
@@ -77,6 +85,7 @@ def renku_op(self):
7785
description=description,
7886
creator=creator,
7987
custom_metadata=custom_metadata,
88+
custom_metadata_source=custom_metadata_source,
8089
keywords=keywords,
8190
)
8291
)

‎renku/ui/service/serializers/datasets.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ class DatasetDetailsRequest(DatasetDetails):
5050

5151
images = fields.List(fields.Nested(ImageObjectRequest))
5252

53-
custom_metadata = fields.Dict()
53+
custom_metadata: fields.Field = fields.Dict()
5454

5555

5656
class DatasetCreateRequest(
@@ -218,7 +218,13 @@ class DatasetEditRequest(
218218
images = fields.List(
219219
fields.Nested(ImageObjectRequest), allow_none=True, metadata={"description": "New dataset images"}
220220
)
221-
custom_metadata = fields.Dict(allow_none=True, metadata={"description": "New custom metadata for the dataset"})
221+
custom_metadata = fields.List(
222+
fields.Dict(), metadata={"description": "New custom metadata for the dataset"}, allow_none=True
223+
)
224+
custom_metadata_source = fields.String(
225+
allow_none=True,
226+
metadata={"description": "Source for the custom metadata for the dataset"},
227+
)
222228

223229

224230
class DatasetEditResponse(RenkuSyncSchema):

‎renku/ui/service/serializers/project.py

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,11 @@ class ProjectShowResponse(Schema):
4444
created = fields.DateTime(metadata={"description": "The date this project was created at."})
4545
creator = fields.Nested(DatasetCreators, metadata={"description": "The creator of this project"})
4646
agent = fields.String(metadata={"description": "The renku version last used on this project"})
47-
custom_metadata = fields.Dict(
48-
dump_default=None, attribute="annotations", metadata={"description": "Custom JSON-LD metadata of the project"}
47+
custom_metadata = fields.List(
48+
fields.Dict(),
49+
dump_default=None,
50+
attribute="annotations",
51+
metadata={"description": "Custom JSON-LD metadata of the project"},
4952
)
5053
template_info = fields.String(
5154
metadata={"description": "The template that was used in the creation of this project"}
@@ -69,7 +72,13 @@ class ProjectEditRequest(AsyncSchema, LocalRepositorySchema, RemoteRepositorySch
6972

7073
description = fields.String(metadata={"description": "New description for the project"})
7174
creator = fields.Nested(DatasetCreators, metadata={"description": "New creator for the project"})
72-
custom_metadata = fields.Dict(allow_none=True, metadata={"description": "New custom JSON-LD metadata"})
75+
custom_metadata = fields.List(
76+
fields.Dict(), metadata={"description": "New custom metadata for the project"}, allow_none=True
77+
)
78+
custom_metadata_source = fields.String(
79+
allow_none=True,
80+
metadata={"description": "The source for the JSON-LD metadata"},
81+
)
7382
keywords = fields.List(fields.String(), allow_none=True, metadata={"description": "New keyword(s) for the project"})
7483

7584

‎tests/cli/test_datasets.py

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1319,6 +1319,70 @@ def test_dataset_edit(runner, project, dirty, subdirectory):
13191319
assert 0 == result.exit_code, format_result_exception(result)
13201320

13211321

1322+
@pytest.mark.parametrize(
1323+
"metadata",
1324+
[
1325+
[
1326+
{
1327+
"@id": "https://example.com/annotation1",
1328+
"@type": "https://schema.org/specialType",
1329+
"https://schema.org/specialProperty": "some_unique_value",
1330+
},
1331+
{
1332+
"@id": "https://example.com/annotation2",
1333+
"@type": "https://schema.org/specialType2",
1334+
"https://schema.org/specialProperty2": "some_unique_value2",
1335+
},
1336+
],
1337+
{
1338+
"@id": "https://example.com/annotation1",
1339+
"@type": "https://schema.org/specialType",
1340+
"https://schema.org/specialProperty": "some_unique_value",
1341+
},
1342+
],
1343+
)
1344+
@pytest.mark.parametrize("source", [None, "test1"])
1345+
def test_dataset_edit_metadata(runner, project, source, metadata):
1346+
"""Check dataset metadata editing."""
1347+
metadata_path = project.path / "metadata.json"
1348+
metadata_path.write_text(json.dumps(metadata))
1349+
create_args = [
1350+
"dataset",
1351+
"create",
1352+
"dataset",
1353+
"-t",
1354+
"original title",
1355+
"-k",
1356+
"keyword-1",
1357+
]
1358+
edit_args = [
1359+
"dataset",
1360+
"edit",
1361+
"dataset",
1362+
"--metadata",
1363+
str(metadata_path),
1364+
]
1365+
if source is None:
1366+
expected_source = "renku"
1367+
else:
1368+
expected_source = source
1369+
edit_args.append("--metadata-source")
1370+
edit_args.append(source)
1371+
result = runner.invoke(cli, create_args)
1372+
assert 0 == result.exit_code, format_result_exception(result)
1373+
result = runner.invoke(cli, edit_args)
1374+
assert 0 == result.exit_code, format_result_exception(result)
1375+
dataset = get_dataset_with_injection("dataset")
1376+
annotation_bodies = [annotation.body for annotation in dataset.annotations]
1377+
annotation_sources = [annotation.source for annotation in dataset.annotations]
1378+
if isinstance(metadata, dict):
1379+
metadata = [metadata]
1380+
assert all([imetadata in annotation_bodies for imetadata in metadata])
1381+
assert all([imetadata in metadata for imetadata in annotation_bodies])
1382+
assert len(annotation_bodies) == len(metadata)
1383+
assert all([isource == expected_source for isource in annotation_sources])
1384+
1385+
13221386
@pytest.mark.parametrize("dirty", [False, True])
13231387
def test_dataset_edit_unset(runner, project, dirty, subdirectory):
13241388
"""Check dataset metadata editing unsetting values."""

0 commit comments

Comments
 (0)