feat(cli): add custom metadata to Plan (#2929)

Author: Panaetius

renku/command/schema/plan.py

@@ -21,7 +21,8 @@
 
 import marshmallow
 
-from renku.command.schema.calamus import JsonLDSchema, Nested, fields, prov, renku, schema
+from renku.command.schema.annotation import AnnotationSchema
+from renku.command.schema.calamus import JsonLDSchema, Nested, fields, oa, prov, renku, schema
 from renku.command.schema.parameter import CommandInputSchema, CommandOutputSchema, CommandParameterSchema
 from renku.domain_model.workflow.plan import Plan
 
@@ -51,6 +52,7 @@ class Meta:
     outputs = Nested(renku.hasOutputs, CommandOutputSchema, many=True, missing=None)
     parameters = Nested(renku.hasArguments, CommandParameterSchema, many=True, missing=None)
     success_codes = fields.List(renku.successCodes, fields.Integer(), missing=[0])
+    annotations = Nested(oa.hasTarget, AnnotationSchema, reverse=True, many=True)
 
     @marshmallow.pre_dump
     def _pre_dump(self, in_data, **kwargs):

renku/command/view_model/plan.py

@@ -17,6 +17,9 @@
 # limitations under the License.
 """Plan view model."""
 
+from __future__ import annotations
+
+import json
 from typing import List, Optional, Union
 
 from renku.domain_model.workflow.composite_plan import CompositePlan
@@ -153,6 +156,7 @@ def __init__(
         parameters: List[CommandParameterViewModel],
         description: Optional[str] = None,
         success_codes: Optional[str] = None,
+        annotations: Optional[str] = None,
     ):
         self.id = id
         self.name = name
@@ -162,6 +166,7 @@ def __init__(
         self.inputs = inputs
         self.outputs = outputs
         self.parameters = parameters
+        self.annotations = annotations
 
     @classmethod
     def from_plan(cls, plan: Plan):
@@ -182,6 +187,9 @@ def from_plan(cls, plan: Plan):
             inputs=[CommandInputViewModel.from_input(input) for input in plan.inputs],
             outputs=[CommandOutputViewModel.from_output(output) for output in plan.outputs],
             parameters=[CommandParameterViewModel.from_parameter(param) for param in plan.parameters],
+            annotations=json.dumps([{"id": a.id, "body": a.body, "source": a.source} for a in plan.annotations])
+            if plan.annotations
+            else None,
         )
 
 

renku/command/workflow.py

@@ -25,6 +25,8 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
 
+from renku.domain_model.provenance.annotation import Annotation
+
 if TYPE_CHECKING:
     from networkx import DiGraph
 
@@ -334,6 +336,7 @@ def _edit_workflow(
     rename_params: List[str],
     describe_params: List[str],
     plan_gateway: IPlanGateway,
+    custom_metadata: Optional[Dict] = None,
 ):
     """Edits a workflow details.
 
@@ -346,6 +349,7 @@ def _edit_workflow(
         rename_params(List[str]): New names for parameters.
         describe_params(List[str]): New descriptions for parameters.
         plan_gateway(IPlanGateway): Injected plan gateway.
+        custom_metadata(Dict, optional): Custom JSON-LD metadata (Default value = None).
 
     Returns:
         Details of the modified Plan.
@@ -359,6 +363,13 @@ def _edit_workflow(
         workflow.description = description
 
     if isinstance(workflow, Plan):
+        if custom_metadata:
+            existing_metadata = [a for a in workflow.annotations if a.source != "renku"]
+
+            existing_metadata.append(Annotation(id=Annotation.generate_id(), body=custom_metadata, source="renku"))
+
+            workflow.annotations = existing_metadata
+
         workflow.set_parameters_from_strings(set_params)
 
         def _mod_params(workflow, changed_params, attr):

renku/core/plugin/run.py

@@ -22,11 +22,13 @@
 
 
 @hookspec
-def process_run_annotations(run):
-    """Plugin Hook to add ``Annotation`` entry list to a ``Activity``.
+def plan_annotations(plan):
+    """Plugin Hook to add ``Annotation`` entry list to a ``Plan``.
+
+    Run when a Plan is created by ``renku run``.
 
     Args:
-        run: A ``Activity`` object to get annotations for.
+        plan: A ``Plan`` object to get annotations for.
 
     Returns:
         A list of ``renku.domain_model.provenance.annotation.Annotation``
@@ -40,23 +42,10 @@ def process_run_annotations(run):
 def activity_annotations(activity):
     """Plugin Hook to add ``Annotation`` entry list to a ``Activity``.
 
-    Args:
-        activity: An ``Activity`` object to get annotations for.
-
-    Returns:
-        A list of ``renku.domain_model.provenance.annotation.Annotation``
-        objects.
-
-    """
-    pass
-
-
-@hookspec
-def cmdline_tool_annotations(tool):
-    """Plugin Hook to add ``Annotation`` entry list to a ``WorkflowTool``.
+    Run when creating an activity from a ``renku run``.
 
     Args:
-        tool:
+        activity: An ``Activity`` object to get annotations for.
 
     Returns:
         A list of ``renku.domain_model.provenance.annotation.Annotation``

renku/core/workflow/plan_factory.py

@@ -34,6 +34,7 @@
 from renku.core.constant import RENKU_HOME, RENKU_TMP
 from renku.core.interface.client_dispatcher import IClientDispatcher
 from renku.core.interface.project_gateway import IProjectGateway
+from renku.core.plugin.pluginmanager import get_plugin_manager
 from renku.core.util.git import is_path_safe
 from renku.core.util.metadata import is_external_file
 from renku.core.util.os import get_absolute_path, get_relative_path, is_subpath
@@ -648,9 +649,6 @@ def watch(self, client_dispatcher: IClientDispatcher, no_output=False):
 
             client.repository.add(*output_paths)
 
-        results = pm.hook.cmdline_tool_annotations(tool=self)
-        self.annotations = [a for r in results for a in r]
-
     def _path_relative_to_root(self, path) -> str:
         """Make a potentially relative path in a subdirectory relative to the root of the repository."""
         absolute_path = get_absolute_path(path, base=self.directory)
@@ -704,7 +702,7 @@ def to_plan(
         keywords: Optional[List[str]] = None,
     ) -> Plan:
         """Return an instance of ``Plan`` based on this factory."""
-        return Plan(
+        plan = Plan(
             id=self.plan_id,
             name=name,
             description=description,
@@ -717,6 +715,15 @@ def to_plan(
             success_codes=self.success_codes,
         )
 
+        pm = get_plugin_manager()
+
+        plugin_annotations = list(chain.from_iterable(pm.hook.plan_annotations(plan=plan)))
+
+        if plugin_annotations:
+            plan.annotations.extend(plugin_annotations)
+
+        return plan
+
 
 def read_files_list(files_list: Path):
     """Read files list yaml containing name:path pairs."""

renku/domain_model/workflow/plan.py

@@ -30,6 +30,7 @@
 
 from renku.core import errors
 from renku.core.util.datetime8601 import local_now
+from renku.domain_model.provenance.annotation import Annotation
 from renku.domain_model.workflow.parameter import CommandInput, CommandOutput, CommandParameter, CommandParameterBase
 from renku.infrastructure.database import Persistent
 
@@ -135,6 +136,8 @@ def is_derivation(self) -> bool:
 class Plan(AbstractPlan):
     """Represent a `renku run` execution template."""
 
+    annotations: List[Annotation] = list()
+
     def __init__(
         self,
         *,
@@ -151,12 +154,14 @@ def __init__(
         project_id: Optional[str] = None,
         outputs: Optional[List[CommandOutput]] = None,
         success_codes: Optional[List[int]] = None,
+        annotations: Optional[List[Annotation]] = None,
     ):
         self.command: str = command
         self.inputs: List[CommandInput] = inputs or []
         self.outputs: List[CommandOutput] = outputs or []
         self.parameters: List[CommandParameter] = parameters or []
         self.success_codes: List[int] = success_codes or []
+        self.annotations: List[Annotation] = annotations or []
         super().__init__(
             id=id,
             description=description,

renku/ui/cli/utils/terminal.py

@@ -70,6 +70,9 @@ def print_description(description):
     print_key_value("Command: ", plan.full_command)
     print_key_value("Success Codes: ", plan.success_codes)
 
+    if plan.annotations:
+        print_key_value("Annotations:\n", plan.annotations)
+
     if plan.inputs:
         print_key("Inputs:")
         for run_input in plan.inputs:

renku/ui/cli/workflow.py

@@ -445,6 +445,30 @@
 rename its parameter ``input-1`` to ``my-input`` and set the default of this
 parameter to ``other-file.txt`` and set its description.
 
++-----------------------+------------------------------------------------------+
+| Option                | Description                                          |
++=======================+======================================================+
+| ``-n, --name``        | Plan's name                                          |
++-----------------------+------------------------------------------------------+
+| ``-d, --description`` | Plan's description.                                  |
++-----------------------+------------------------------------------------------+
+| ``-s, --set``         | Set default value for a parameter.                   |
+|                       | Accepted format is '<name>=<value>'                  |
++-----------------------+------------------------------------------------------+
+| ``-m, --map``         | Add a new mapping on the Plan.                       |
+|                       | Accepted format is '<name>=<name or expression>'     |
++-----------------------+------------------------------------------------------+
+| ``-r, --rename-param``| Rename a parameter.                                  |
+|                       | Accepted format is '<name>="new name"'               |
++-----------------------+------------------------------------------------------+
+| ``-d,``               | Add a description for a parameter.                   |
+| ``--describe-param``  | Accepted format is '<name>="description"'            |
++-----------------------+------------------------------------------------------+
+| ``-m, --metadata``    | Path to file containing custom JSON-LD metadata to   |
+|                       | be added to the dataset.                             |
++-----------------------+------------------------------------------------------+
+
+
 .. cheatsheet::
    :group: Workflows
    :command: $ renku workflow edit <plan>
@@ -636,7 +660,7 @@
 
 import renku.ui.cli.utils.color as color
 from renku.command.echo import ERROR
-from renku.command.format.workflow import WORKFLOW_COLUMNS, WORKFLOW_FORMATS
+from renku.command.format.workflow import WORKFLOW_COLUMNS, WORKFLOW_FORMATS, json
 from renku.command.view_model.activity_graph import ACTIVITY_GRAPH_COLUMNS
 from renku.core import errors
 from renku.ui.cli.utils.callback import ClickCallback
@@ -837,23 +861,26 @@ def compose(
 
 @workflow.command()
 @click.argument("workflow_name", metavar="<name or uuid>", shell_complete=_complete_workflows)
-@click.option("--name", metavar="<new name>", help="New name of the workflow")
-@click.option("--description", metavar="<new desc>", help="New description of the workflow")
+@click.option("-n", "--name", metavar="<new name>", help="New name of the workflow")
+@click.option("-d", "--description", metavar="<new desc>", help="New description of the workflow")
 @click.option(
+    "-s",
     "--set",
     "set_params",
     multiple=True,
     metavar="<parameter>=<value>",
     help="Set default <value> for a <parameter>/add new parameter",
 )
 @click.option(
+    "-m",
     "--map",
     "map_params",
     multiple=True,
     metavar="<parameter>=<parameter or expression>",
     help="New mapping on the workflow",
 )
 @click.option(
+    "-r",
     "--rename-param",
     "rename_params",
     multiple=True,
@@ -867,12 +894,24 @@ def compose(
     metavar='<parameter>="description"',
     help="New description of the workflow",
 )
-def edit(workflow_name, name, description, set_params, map_params, rename_params, describe_params):
+@click.option(
+    "-m",
+    "--metadata",
+    default=None,
+    type=click.Path(exists=True, dir_okay=False),
+    help="Custom metadata to be associated with the workflow.",
+)
+def edit(workflow_name, name, description, set_params, map_params, rename_params, describe_params, metadata):
     """Edit workflow details."""
     from renku.command.view_model.plan import PlanViewModel
     from renku.command.workflow import edit_workflow_command
     from renku.ui.cli.utils.terminal import print_plan
 
+    custom_metadata = None
+
+    if metadata:
+        custom_metadata = json.loads(Path(metadata).read_text())
+
     result = (
         edit_workflow_command()
         .build()
@@ -884,6 +923,7 @@ def edit(workflow_name, name, description, set_params, map_params, rename_params
             map_params=map_params,
             rename_params=rename_params,
             describe_params=describe_params,
+            custom_metadata=custom_metadata,
         )
     )
     if not result.error:

tests/core/fixtures/core_plugins.py

@@ -26,52 +26,54 @@
 
 
 @pytest.fixture
-def dummy_run_plugin_hook():
+def dummy_pre_run_plugin_hook():
     """A dummy hook to be used with the renku run plugin."""
     from renku.core.plugin import hookimpl
 
-    class _CmdlineToolAnnotations(object):
+    class _PreRun(object):
         """CmdlineTool Hook implementation namespace."""
 
-        @hookimpl
-        def cmdline_tool_annotations(self, tool):
-            """``cmdline_tool_annotations`` hook implementation."""
-            from renku.domain_model.provenance.annotation import Annotation
+        called = 0
 
-            return [Annotation(id="_:annotation", source="Dummy Cmdline Hook", body="dummy cmdline hook body")]
+        @hookimpl
+        def pre_run(self, tool):
+            """``pre_run`` hook implementation."""
+            self.called = 1
 
-    return _CmdlineToolAnnotations()
+    return _PreRun()
 
 
 @pytest.fixture
-def dummy_pre_run_plugin_hook():
-    """A dummy hook to be used with the renku run plugin."""
+def dummy_plan_plugin_hook():
+    """A dummy hook to be used with the renku run plugin for plans."""
     from renku.core.plugin import hookimpl
 
-    class _PreRun(object):
+    class _PlanPlugin(object):
         """CmdlineTool Hook implementation namespace."""
 
         called = 0
 
         @hookimpl
-        def pre_run(self, tool):
-            """``cmdline_tool_annotations`` hook implementation."""
-            self.called = 1
+        def plan_annotations(self, plan):
+            """``plan_annotations`` hook implementation."""
+            from renku.domain_model.provenance.annotation import Annotation
 
-    return _PreRun()
+            return [Annotation(id="_:annotation", source="Dummy Plan Hook", body="dummy Plan hook body")]
+
+    return _PlanPlugin()
 
 
 @pytest.fixture
-def dummy_processrun_plugin_hook():
+def dummy_activity_plugin_hook():
     """A dummy hook to be used with the renku run plugin."""
     from renku.core.plugin import hookimpl
 
     class _ActivityAnnotations(object):
         """CmdlineTool Hook implementation namespace."""
 
         @hookimpl
-        def process_run_annotations(self, plan):
-            """``process_run_annotations`` hook implementation."""
+        def activity_annotations(self, activity):
+            """``activity_annotations`` hook implementation."""
             from renku.domain_model.provenance.annotation import Annotation
 
             return [Annotation(id="_:annotation", source="Dummy Activity Hook", body="dummy Activity hook body")]