feat(core): allow partial updates on dataset and project edit (#2949)

Author: Panaetius

docs/conf.py

@@ -356,6 +356,7 @@
     ("py:class", "IDatasetGateway"),
     ("py:class", "IPlanGateway"),
     ("py:class", "LocalClient"),
+    ("py:class", "NoValueType"),
     ("py:class", "OID_TYPE"),
     ("py:class", "Path"),
     ("py:class", "Persistent"),

renku/command/project.py

@@ -17,24 +17,34 @@
 # limitations under the License.
 """Project management."""
 
+from typing import Dict, List, Optional, Union, cast
+
 from renku.command.command_builder import inject
 from renku.command.command_builder.command import Command
 from renku.command.view_model.project import ProjectViewModel
 from renku.core.interface.client_dispatcher import IClientDispatcher
 from renku.core.interface.project_gateway import IProjectGateway
 from renku.core.management.repository import DATABASE_METADATA_PATH
 from renku.core.util.metadata import construct_creator
+from renku.core.util.util import NO_VALUE, NoValueType
+from renku.domain_model.provenance.agent import Person
 
 
 @inject.autoparams()
-def _edit_project(description, creator, keywords, custom_metadata, project_gateway: IProjectGateway):
+def _edit_project(
+    description: Optional[Union[str, NoValueType]],
+    creator: Union[Dict, str, NoValueType],
+    keywords: Optional[Union[List[str], NoValueType]],
+    custom_metadata: Optional[Union[Dict, NoValueType]],
+    project_gateway: IProjectGateway,
+):
     """Edit dataset metadata.
 
     Args:
-        description: New description.
-        creator: New creators.
-        keywords: New keywords.
-        custom_metadata: Custom JSON-LD metadata.
+        description(Union[Optional[str], NoValueType]): New description.
+        creator(Union[Dict, str, NoValueType]): New creators.
+        keywords(Union[Optional[List[str]]): New keywords.
+        custom_metadata(Union[Optional[Dict]): Custom JSON-LD metadata.
         project_gateway(IProjectGateway): Injected project gateway.
 
     Returns:
@@ -47,14 +57,18 @@ def _edit_project(description, creator, keywords, custom_metadata, project_gatew
         "custom_metadata": custom_metadata,
     }
 
-    creator, no_email_warnings = construct_creator(creator, ignore_email=True)
+    no_email_warnings: Optional[Union[Dict, str]] = None
+    parsed_creator: Optional[Union[NoValueType, Person]] = NO_VALUE
+
+    if creator is not NO_VALUE:
+        parsed_creator, no_email_warnings = construct_creator(cast(Union[Dict, str], creator), ignore_email=True)
 
-    updated = {k: v for k, v in possible_updates.items() if v}
+    updated = {k: v for k, v in possible_updates.items() if v is not NO_VALUE}
 
     if updated:
         project = project_gateway.get_project()
         project.update_metadata(
-            creator=creator, description=description, keywords=keywords, custom_metadata=custom_metadata
+            creator=parsed_creator, description=description, keywords=keywords, custom_metadata=custom_metadata
         )
         project_gateway.update_project(project)
 

renku/core/dataset/dataset.py

@@ -23,7 +23,7 @@
 import urllib
 from collections import OrderedDict
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
 
 import patoolib
 
@@ -48,6 +48,7 @@
 from renku.core.util.os import delete_file
 from renku.core.util.tabulate import tabulate
 from renku.core.util.urls import get_slug, remove_credentials
+from renku.core.util.util import NO_VALUE, NoValueType
 from renku.domain_model.dataset import (
     Dataset,
     DatasetDetailsJson,
@@ -175,63 +176,68 @@ def create_dataset(
 @inject.autoparams("client_dispatcher")
 def edit_dataset(
     name: str,
-    title: str,
-    description: str,
-    creators: List[Person],
+    title: Optional[Union[str, NoValueType]],
+    description: Optional[Union[str, NoValueType]],
+    creators: Optional[Union[List[Person], NoValueType]],
     client_dispatcher: IClientDispatcher,
-    keywords: Optional[List[str]] = None,
-    images: Optional[List[ImageRequestModel]] = None,
-    skip_image_update: bool = False,
-    custom_metadata: Optional[Dict] = None,
+    keywords: Optional[Union[List[str], NoValueType]] = NO_VALUE,
+    images: Optional[Union[List[ImageRequestModel], NoValueType]] = NO_VALUE,
+    custom_metadata: Optional[Union[Dict, NoValueType]] = NO_VALUE,
 ):
     """Edit dataset metadata.
 
     Args:
         name(str): Name of the dataset to edit
-        title(str): New title for the dataset.
-        description(str): New description for the dataset.
-        creators(List[Person]): New creators for the dataset.
+        title(Optional[Union[str, NoValueType]]): New title for the dataset.
+        description(Optional[Union[str, NoValueType]]): New description for the dataset.
+        creators(Optional[Union[List[Person], NoValueType]]): New creators for the dataset.
         client_dispatcher(IClientDispatcher): Injected client dispatcher.
-        keywords(List[str], optional): New keywords for dataset (Default value = None).
-        images(List[ImageRequestModel], optional): New images for dataset (Default value = None).
-        skip_image_update(bool, optional): Whether or not to skip updating dataset images (Default value = False).
-        custom_metadata(Dict, optional): Custom JSON-LD metadata (Default value = None).
+        keywords(Optional[Union[List[str], NoValueType]]): New keywords for dataset (Default value = ``NO_VALUE``).
+        images(Optional[Union[List[ImageRequestModel], NoValueType]]): New images for dataset
+            (Default value = ``NO_VALUE``).
+        custom_metadata(Optional[Union[Dict, NoValueType]]): Custom JSON-LD metadata (Default value = ``NO_VALUE``).
 
     Returns:
         bool: True if updates were performed.
     """
     client = client_dispatcher.current_client
 
+    if isinstance(title, str):
+        title = title.strip()
+
+    if title is None:
+        title = ""
+
     possible_updates = {
         "creators": creators,
         "description": description,
         "keywords": keywords,
         "title": title,
     }
 
-    title = title.strip() if isinstance(title, str) else ""
-
     dataset_provenance = DatasetsProvenance()
     dataset = dataset_provenance.get_by_name(name=name)
 
     if dataset is None:
         raise errors.ParameterError("Dataset does not exist.")
 
-    updated: Dict[str, Any] = {k: v for k, v in possible_updates.items() if v}
+    updated: Dict[str, Any] = {k: v for k, v in possible_updates.items() if v != NO_VALUE}
 
     if updated:
         dataset.update_metadata(creators=creators, description=description, keywords=keywords, title=title)
 
-    if skip_image_update:
+    if images == NO_VALUE:
         images_updated = False
     else:
-        images_updated = set_dataset_images(client, dataset, images)
+        images_updated = set_dataset_images(client, dataset, cast(Optional[List[ImageRequestModel]], images))
 
     if images_updated:
-        updated["images"] = [{"content_url": i.content_url, "position": i.position} for i in dataset.images]
+        updated["images"] = (
+            None if images is None else [{"content_url": i.content_url, "position": i.position} for i in dataset.images]
+        )
 
-    if custom_metadata:
-        update_dataset_custom_metadata(dataset, custom_metadata)
+    if custom_metadata is not NO_VALUE:
+        update_dataset_custom_metadata(dataset, cast(Optional[Dict], custom_metadata))
         updated["custom_metadata"] = custom_metadata
 
     if not updated:
@@ -840,7 +846,7 @@ def set_dataset_images(client: "LocalClient", dataset: Dataset, images: Optional
     return images_updated or dataset.images != previous_images
 
 
-def update_dataset_custom_metadata(dataset: Dataset, custom_metadata: Dict):
+def update_dataset_custom_metadata(dataset: Dataset, custom_metadata: Optional[Dict]):
     """Update custom metadata on a dataset.
 
     Args:
@@ -850,7 +856,8 @@ def update_dataset_custom_metadata(dataset: Dataset, custom_metadata: Dict):
 
     existing_metadata = [a for a in dataset.annotations if a.source != "renku"]
 
-    existing_metadata.append(Annotation(id=Annotation.generate_id(), body=custom_metadata, source="renku"))
+    if custom_metadata is not None:
+        existing_metadata.append(Annotation(id=Annotation.generate_id(), body=custom_metadata, source="renku"))
 
     dataset.annotations = existing_metadata
 

renku/core/util/metadata.py

@@ -34,7 +34,7 @@
     from renku.domain_model.provenance.agent import Person
 
 
-def construct_creators(creators: List[Union[dict, str]], ignore_email=False):
+def construct_creators(creators: Optional[List[Union[dict, str]]], ignore_email=False):
     """Parse input and return a list of Person."""
     creators = creators or []
 

renku/core/util/util.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 """General utility functions."""
 
-from typing import Any, Optional
+from typing import Any, NewType, Optional
 
 from packaging.version import Version
 
@@ -33,3 +33,10 @@ def to_semantic_version(value: str) -> Optional[Version]:
         return Version(value)
     except ValueError:
         return None
+
+
+NoValueType = NewType("NoValueType", object)
+"""Type to represent a value not being set in cases where ``None`` is a valid value."""
+
+NO_VALUE = NoValueType(object())
+"""Sentinel to represent a value not being set in cases where ``None`` is a valid value."""

renku/domain_model/dataset.py

@@ -33,6 +33,7 @@
 from renku.core.util.git import get_entity_from_revision
 from renku.core.util.metadata import is_external_file
 from renku.core.util.urls import get_path, get_slug
+from renku.core.util.util import NO_VALUE
 from renku.infrastructure.immutable import Immutable, Slots
 from renku.infrastructure.persistent import Persistent
 
@@ -554,7 +555,7 @@ def update_metadata(self, **kwargs):
         for name, value in kwargs.items():
             if name not in editable_attributes:
                 raise errors.ParameterError(f"Cannot edit field: '{name}'")
-            if value and value != getattr(self, name):
+            if value is not NO_VALUE and value != getattr(self, name):
                 setattr(self, name, value)
 
     def unlink_file(self, path, missing_ok=False) -> Optional[DatasetFile]:

renku/domain_model/project.py

@@ -27,6 +27,7 @@
 from renku.core.util.datetime8601 import fix_datetime, local_now, parse_date
 from renku.core.util.git import get_git_user
 from renku.core.util.os import normalize_to_ascii
+from renku.core.util.util import NO_VALUE
 from renku.domain_model.provenance.agent import Person
 from renku.domain_model.provenance.annotation import Annotation
 from renku.version import __minimum_project_version__
@@ -155,12 +156,13 @@ def update_metadata(self, custom_metadata=None, **kwargs):
         for name, value in kwargs.items():
             if name not in editable_attributes:
                 raise errors.ParameterError(f"Cannot edit field: '{name}'")
-            if value and value != getattr(self, name):
+            if value is not NO_VALUE and value != getattr(self, name):
                 setattr(self, name, value)
 
-        if custom_metadata:
+        if custom_metadata is not NO_VALUE:
             existing_metadata = [a for a in self.annotations if a.source != "renku"]
 
-            existing_metadata.append(Annotation(id=Annotation.generate_id(), body=custom_metadata, source="renku"))
+            if custom_metadata is not None:
+                existing_metadata.append(Annotation(id=Annotation.generate_id(), body=custom_metadata, source="renku"))
 
             self.annotations = existing_metadata

renku/ui/cli/dataset.py

@@ -67,11 +67,13 @@
 
 Use the ``edit`` sub-command to change metadata of a dataset. You can edit the same
 set of metadata as the create command by passing the options described in the
-table above.
+table above. You can also use the ``-u/--unset`` options with one of the values
+from ``creators`` (short ``c``), ``keywords`` (short ``k``) or ``metadata``
+(short ``m``) to delete the respective values from the dataset.
 
 .. code-block:: console
 
-    $ renku dataset edit my-dataset --title 'New title'
+    $ renku dataset edit my-dataset --title 'New title' --unset keywords
     Successfully updated: title.
 
 Listing all datasets:
@@ -543,6 +545,7 @@
 from renku.command.format.dataset_files import DATASET_FILES_COLUMNS, DATASET_FILES_FORMATS
 from renku.command.format.dataset_tags import DATASET_TAGS_FORMATS
 from renku.command.format.datasets import DATASETS_COLUMNS, DATASETS_FORMATS
+from renku.core.util.util import NO_VALUE
 
 
 def _complete_datasets(ctx, param, incomplete):
@@ -644,40 +647,79 @@ def create(name, title, description, creators, metadata, keyword):
 
 @dataset.command()
 @click.argument("name", shell_complete=_complete_datasets)
-@click.option("-t", "--title", default=None, type=click.STRING, help="Title of the dataset.")
-@click.option("-d", "--description", default=None, type=click.STRING, help="Dataset's description.")
+@click.option("-t", "--title", default=NO_VALUE, type=click.UNPROCESSED, help="Title of the dataset.")
+@click.option("-d", "--description", default=NO_VALUE, type=click.UNPROCESSED, help="Dataset's description.")
 @click.option(
     "-c",
     "--creator",
     "creators",
-    default=None,
+    default=[NO_VALUE],
     multiple=True,
+    type=click.UNPROCESSED,
     help="Creator's name, email, and affiliation. " "Accepted format is 'Forename Surname <email> [affiliation]'.",
 )
 @click.option(
     "-m",
     "--metadata",
-    default=None,
-    type=click.Path(exists=True, dir_okay=False),
+    default=NO_VALUE,
+    type=click.UNPROCESSED,
     help="Custom metadata to be associated with the dataset.",
 )
-@click.option("-k", "--keyword", default=None, multiple=True, type=click.STRING, help="List of keywords or tags.")
-def edit(name, title, description, creators, metadata, keyword):
+@click.option(
+    "-k",
+    "--keyword",
+    "keywords",
+    default=[NO_VALUE],
+    multiple=True,
+    type=click.UNPROCESSED,
+    help="List of keywords or tags.",
+)
+@click.option(
+    "-u",
+    "--unset",
+    default=[],
+    multiple=True,
+    type=click.Choice(["keywords", "k", "images", "i", "metadata", "m"]),
+    help="Remove keywords from dataset.",
+)
+def edit(name, title, description, creators, metadata, keywords, unset):
     """Edit dataset metadata."""
     from renku.command.dataset import edit_dataset_command
     from renku.core.util.metadata import construct_creators
     from renku.ui.cli.utils.callback import ClickCallback
 
-    creators = creators or ()
-    keywords = keyword or ()
+    images = NO_VALUE
 
-    custom_metadata = None
+    if list(creators) == [NO_VALUE]:
+        creators = NO_VALUE
+
+    if list(keywords) == [NO_VALUE]:
+        keywords = NO_VALUE
+
+    if "k" in unset or "keywords" in unset:
+        if keywords is not NO_VALUE:
+            raise click.UsageError("Cant use '--keyword' together with unsetting keyword")
+        keywords = None
+
+    if "m" in unset or "metadata" in unset:
+        if metadata is not NO_VALUE:
+            raise click.UsageError("Cant use '--metadata' together with unsetting metadata")
+        metadata = None
+
+    if "i" in unset or "images" in unset:
+        images = None
+
+    custom_metadata = metadata
     no_email_warnings = False
 
-    if creators:
+    if creators and creators is not NO_VALUE:
         creators, no_email_warnings = construct_creators(creators, ignore_email=True)
 
-    if metadata:
+    if metadata and metadata is not NO_VALUE:
+        path = Path(metadata)
+
+        if not path.exists():
+            raise click.UsageError(f"Path {path} does not exist.")
         custom_metadata = json.loads(Path(metadata).read_text())
 
     updated = (
@@ -689,7 +731,7 @@ def edit(name, title, description, creators, metadata, keyword):
             description=description,
             creators=creators,
             keywords=keywords,
-            skip_image_update=True,
+            images=images,
             custom_metadata=custom_metadata,
         )
     ).output