feat(workflow): support for workflow definition files (#3176)

Author: mohammad-alisafaee

conftest.py

@@ -42,6 +42,7 @@
 
 GLOBAL_FIXTURE_LOCATIONS = [
     "tests.fixtures.common",
+    "tests.fixtures.communication",
     "tests.fixtures.config",
     "tests.fixtures.domain_models",
     "tests.fixtures.repository",

docs/cheatsheet/conf.py

@@ -196,6 +196,7 @@
     ("py:class", "OID_TYPE"),
     ("py:class", "Path"),
     ("py:class", "Persistent"),
+    ("py:class", "WorkflowFileCompositePlan"),
     ("py:class", "optional"),
     ("py:class", '"ValueResolver"'),
     ("py:exc", "errors.ParameterError"),

docs/conf.py

@@ -363,12 +363,15 @@
     ("py:class", "OID_TYPE"),
     ("py:class", "Path"),
     ("py:class", "Persistent"),
+    ("py:class", "WorkflowFileCompositePlan"),
+    ("py:class", "itertools.count"),
     ("py:class", "optional"),
     ("py:class", '"ValueResolver"'),
     ("py:exc", "errors.ParameterError"),
 ]
 
 nitpick_ignore_regex = [
+    ("py:class", r"bashlex.*"),
     ("py:class", r"calamus.*"),
     ("py:class", r"docker.*"),
     ("py:class", r"marshmallow.*"),

docs/reference/core.rst

@@ -77,6 +77,10 @@ Schema classes used to serialize domain models to JSON-LD.
    :members:
    :show-inheritance:
 
+.. automodule:: renku.command.schema.workflow_file
+   :members:
+   :show-inheritance:
+
 
 Datasets
 --------
@@ -178,7 +182,11 @@ Workflows
    :members:
    :show-inheritance:
 
-.. automodule:: renku.core.workflow.concrete_execution_graph
+.. automodule:: renku.core.workflow.model.concrete_execution_graph
+   :members:
+   :show-inheritance:
+
+.. automodule:: renku.core.workflow.model.workflow_file
    :members:
    :show-inheritance:
 
@@ -193,6 +201,10 @@ Workflows
    :members:
    :show-inheritance:
 
+.. automodule:: renku.core.workflow.workflow_file
+   :members:
+   :show-inheritance:
+
 
 Sessions
 --------

docs/reference/models/workflow.rst

@@ -44,6 +44,13 @@ parameter
    :members:
    :show-inheritance:
 
+workflow\_file
+---------------------------------------------------
+
+.. automodule:: renku.domain_model.workflow.workflow_file
+   :members:
+   :show-inheritance:
+
 
 Renku Workflow Conversion
 =========================

docs/spelling_wordlist.txt

@@ -263,6 +263,7 @@ typesystem
 Ubuntu
 Unmount
 ui
+Unescape
 unhandled
 unicode
 unlink

pyproject.toml

@@ -55,6 +55,7 @@ apispec = { version = ">=4.0.0,<5.3.0", optional = true }
 apispec-webframeworks = { version = "<0.6,>=0.5.2", optional = true }
 appdirs = "<=1.4.4,>=1.4.3"
 attrs = ">=21.1.0,<22.2.0"
+bashlex = ">=0.16,<0.17"
 black = { version = "==22.6.0", optional = true }
 calamus = ">=0.3.13,<0.5"
 circus = { version = "==0.17.1", optional = true }
@@ -322,7 +323,7 @@ addopts = "--flake8 --black --doctest-glob=\"*.rst\" --doctest-modules --cov --c
 doctest_optionflags = "ALLOW_UNICODE"
 flake8-ignore = ["*.py", "E121", "E126", "E203", "E226", "E231", "W503", "W504", "docs/conf.py", "docs/cheatsheet/conf.py", "ALL"]
 flake8-max-line-length = 120
-testpaths = ["docs", "tests", "conftest.py"]
+testpaths = ["docs", "tests", "renku", "conftest.py"]
 markers = [
     "integration: mark a test as a integration.",
     "jobs: mark a test as a job test.",
@@ -353,6 +354,7 @@ module = [
     "apispec_webframeworks.*",
     "appdirs",
     "BTrees.*",
+    "bashlex.*",
     "calamus.*",
     "deepdiff",
     "deepmerge",

renku/command/checks/activities.py

@@ -48,7 +48,9 @@ def check_migrated_activity_ids(fix, activity_gateway: IActivityGateway, **_):
             activity._p_oid = current_database.hash_id(activity.id)
             activity.freeze()
 
-            for attribute in itertools.chain(activity.usages, activity.generations, activity.parameters):
+            for attribute in itertools.chain(
+                activity.usages, activity.hidden_usages, activity.generations, activity.parameters
+            ):
                 object.__setattr__(attribute, "id", f"/activities/{attribute.id}")  # type: ignore
 
             activity.association.id = f"/activities/{activity.association.id}"

renku/command/command_builder/command.py

@@ -404,6 +404,7 @@ def with_commit(
         raise_if_empty: bool = False,
         commit_only: Optional[Union[str, List[Union[str, Path]]]] = None,
         skip_staging: bool = False,
+        skip_dirty_checks: bool = False,
     ) -> "Command":
         """Create a commit.
 
@@ -413,10 +414,20 @@ def with_commit(
             raise_if_empty(bool, optional): Whether to raise an exception if there are no modified files
                 (Default value = False).
             commit_only(bool, optional): Only commit the supplied paths (Default value = None).
+            skip_staging(bool): Don't commit staged files.
+            skip_dirty_checks(bool): Don't check if paths are dirty or staged.
         """
         from renku.command.command_builder.repo import Commit
 
-        return Commit(self, message, commit_if_empty, raise_if_empty, commit_only, skip_staging)
+        return Commit(
+            self,
+            message=message,
+            commit_if_empty=commit_if_empty,
+            raise_if_empty=raise_if_empty,
+            commit_only=commit_only,
+            skip_staging=skip_staging,
+            skip_dirty_checks=skip_dirty_checks,
+        )
 
     @check_finalized
     def lock_project(self) -> "Command":

renku/command/command_builder/repo.py

@@ -39,6 +39,7 @@ def __init__(
         raise_if_empty: Optional[bool] = False,
         commit_only: Optional[Union[str, List[Union[str, Path]]]] = None,
         skip_staging: bool = False,
+        skip_dirty_checks: bool = False,
     ) -> None:
         """__init__ of Commit.
 
@@ -47,13 +48,16 @@ def __init__(
             commit_if_empty (bool): Whether to commit if there are no modified files (Default value = None).
             raise_if_empty (bool): Whether to raise an exception if there are no modified files (Default value = None).
             commit_only (bool): Only commit the supplied paths (Default value = None).
+            skip_staging(bool): Don't commit staged files.
+            skip_dirty_checks(bool): Don't check if paths are dirty or staged.
         """
         self._builder = builder
         self._message = message
         self._commit_if_empty = commit_if_empty
         self._raise_if_empty = raise_if_empty
         self._commit_filter_paths = commit_only
         self._skip_staging: bool = skip_staging
+        self._skip_dirty_checks: bool = skip_dirty_checks
 
     def _pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
         """Hook to create a commit transaction.
@@ -71,6 +75,7 @@ def _pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
             repository=project_context.repository,
             commit_only=self._commit_filter_paths,
             skip_staging=self._skip_staging,
+            skip_dirty_checks=self._skip_dirty_checks,
         )
 
     def _post_hook(self, builder: Command, context: dict, result: CommandResult, *args, **kwargs):