feat(cli): gc command (#2866)

mohammad-alisafaee · web-flow · commit 0d3c17653e1f · 2022-04-29T15:02:46.000+04:30
diff --git a/docs/_static/cheatsheet/cheatsheet.pdf b/docs/_static/cheatsheet/cheatsheet.pdf
diff --git a/docs/cheatsheet_hash b/docs/cheatsheet_hash
@@ -1,2 +1,2 @@
-8f835f7c5c76b60f8cd9785fd1e0cf31  cheatsheet.tex
+3e7d7567c17eba64945b70a3c1009343  cheatsheet.tex
 c70c179e07f04186ec05497564165f11  sdsc_cheatsheet.cls
diff --git a/docs/reference/commands.rst b/docs/reference/commands.rst
@@ -62,6 +62,13 @@ Renku Command Line
 
 .. automodule:: renku.ui.cli.dataset
 
+.. _cli-gc:
+
+``renku gc``
+------------
+
+.. automodule:: renku.ui.cli.gc
+
 .. _cli-graph:
 
 ``renku graph``
diff --git a/renku/command/command_builder/client_dispatcher.py b/renku/command/command_builder/client_dispatcher.py
@@ -17,11 +17,11 @@
 # limitations under the License.
 """Renku client dispatcher."""
 from pathlib import Path
-from typing import Optional, Union
+from typing import Union
 
 from renku.core import errors
+from renku.core.constant import RENKU_HOME
 from renku.core.interface.client_dispatcher import IClientDispatcher
-from renku.core.management import RENKU_HOME
 from renku.core.management.client import LocalClient
 
 
@@ -35,7 +35,7 @@ def __init__(self):
         self.client_stack = []
 
     @property
-    def current_client(self) -> Optional[LocalClient]:
+    def current_client(self) -> LocalClient:
         """Get the currently active client."""
         if len(self.client_stack) == 0:
             raise errors.ConfigurationError("No client configured for injection")
diff --git a/renku/command/gc.py b/renku/command/gc.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Garbage collection and cleanup command."""
+
+from renku.command.command_builder.command import Command
+
+
+def gc_command():
+    """Command to clean up project's caches."""
+    from renku.core.misc.gc import remove_caches
+
+    return Command().command(remove_caches).lock_project()
diff --git a/renku/command/init.py b/renku/command/init.py
@@ -26,10 +26,10 @@
 from renku.command.command_builder.command import Command, inject
 from renku.command.git import set_git_home
 from renku.core import errors
+from renku.core.constant import RENKU_HOME
 from renku.core.interface.client_dispatcher import IClientDispatcher
 from renku.core.interface.database_dispatcher import IDatabaseDispatcher
 from renku.core.interface.database_gateway import IDatabaseGateway
-from renku.core.management import RENKU_HOME
 from renku.core.migration.utils import OLD_METADATA_PATH
 from renku.core.template.template import (
     FileAction,
diff --git a/renku/core/constant.py b/renku/core/constant.py
@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2017-2022 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Renku core constants."""
+
+CACHE = "cache"
+"""Directory to cache transient data."""
+
+RENKU_HOME = ".renku"
+"""Project directory name."""
+
+RENKU_TMP = "tmp"
diff --git a/renku/core/dataset/constant.py b/renku/core/dataset/constant.py
@@ -19,16 +19,13 @@
 
 from pathlib import Path
 
-from renku.core.management import RENKU_HOME
+from renku.core.constant import RENKU_HOME
 from renku.core.management.repository import RepositoryApiMixin
 from renku.domain_model.refs import LinkReference
 
 POINTERS = "pointers"
 """Directory for storing external pointer files."""
 
-CACHE = "cache"
-"""Directory to cache transient data."""
-
 DATASET_IMAGES = "dataset_images"
 """Directory for dataset images."""
 
diff --git a/renku/core/dataset/dataset_add.py b/renku/core/dataset/dataset_add.py
@@ -30,7 +30,8 @@
 
 from renku.command.command_builder.command import inject
 from renku.core import errors
-from renku.core.dataset.constant import CACHE, renku_pointers_path
+from renku.core.constant import CACHE
+from renku.core.dataset.constant import renku_pointers_path
 from renku.core.dataset.context import DatasetContext
 from renku.core.dataset.datasets_provenance import DatasetsProvenance
 from renku.core.dataset.pointer_file import create_external_file
diff --git a/renku/core/errors.py b/renku/core/errors.py
@@ -23,7 +23,7 @@
 
 import click
 
-from renku.core.management import RENKU_HOME
+from renku.core.constant import RENKU_HOME
 
 
 class RenkuException(Exception):
diff --git a/renku/core/interface/client_dispatcher.py b/renku/core/interface/client_dispatcher.py
@@ -19,7 +19,10 @@
 
 from abc import ABC
 from pathlib import Path
-from typing import Union
+from typing import TYPE_CHECKING, Union
+
+if TYPE_CHECKING:
+    from renku.core.management.client import LocalClient
 
 
 class IClientDispatcher(ABC):
@@ -29,7 +32,7 @@ class IClientDispatcher(ABC):
     """
 
     @property
-    def current_client(self):
+    def current_client(self) -> "LocalClient":
         """Get the currently active client."""
         raise NotImplementedError
 
diff --git a/renku/core/management/__init__.py b/renku/core/management/__init__.py
@@ -16,6 +16,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Renku repository management."""
-
-RENKU_HOME = ".renku"
-"""Project directory name."""
diff --git a/renku/core/management/config.py b/renku/core/management/config.py
@@ -25,7 +25,7 @@
 import click
 import portalocker
 
-from renku.core.management import RENKU_HOME
+from renku.core.constant import RENKU_HOME
 from renku.domain_model.enums import ConfigFilter
 
 APP_NAME = "Renku"
diff --git a/renku/core/management/migrate.py b/renku/core/management/migrate.py
@@ -37,6 +37,7 @@
 from packaging.version import Version
 
 from renku.command.command_builder.command import inject
+from renku.core.constant import RENKU_TMP
 from renku.core.errors import (
     DockerfileUpdateError,
     MigrationError,
@@ -55,7 +56,6 @@
     read_project_version,
 )
 from renku.core.util import communication
-from renku.core.workflow.plan_factory import RENKU_TMP
 
 try:
     import importlib_resources
@@ -165,7 +165,7 @@ def migrate(
 
 
 def _remove_untracked_renku_files(renku_path):
-    from renku.core.dataset.constant import CACHE
+    from renku.core.constant import CACHE
 
     untracked_paths = [RENKU_TMP, CACHE, "vendors"]
     for path in untracked_paths:
diff --git a/renku/core/management/repository.py b/renku/core/management/repository.py
@@ -30,9 +30,9 @@
 from renku.command.command_builder import inject
 from renku.core import errors
 from renku.core.compat import Path
+from renku.core.constant import RENKU_HOME
 from renku.core.interface.database_gateway import IDatabaseGateway
 from renku.core.interface.project_gateway import IProjectGateway
-from renku.core.management import RENKU_HOME
 from renku.core.management.git import GitCore
 from renku.core.util.git import default_path
 from renku.domain_model.enums import ConfigFilter
diff --git a/renku/core/migration/m_0003__2_initial.py b/renku/core/migration/m_0003__2_initial.py
@@ -21,7 +21,7 @@
 import urllib
 from pathlib import Path
 
-from renku.core.management import RENKU_HOME
+from renku.core.constant import RENKU_HOME
 from renku.core.management.repository import DEFAULT_DATA_DIR as DATA_DIR
 from renku.core.migration.models.v3 import Collection, Dataset, Project, get_client_datasets
 from renku.core.migration.models.v9 import generate_file_id, generate_label
diff --git a/renku/core/misc/__init__.py b/renku/core/misc/__init__.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2017-2022 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Various management functionalities."""
diff --git a/renku/core/misc/gc.py b/renku/core/misc/gc.py
@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Project cleanup management."""
+
+from renku.command.command_builder.command import inject
+from renku.core.constant import CACHE, RENKU_HOME, RENKU_TMP
+from renku.core.interface.client_dispatcher import IClientDispatcher
+
+
+@inject.autoparams()
+def remove_caches(client_dispatcher: IClientDispatcher):
+    """Remove caches and temporary files.
+
+    Args:
+        client_dispatcher(IClientDispatcher):  Injected client dispatcher.
+
+    """
+    client = client_dispatcher.current_client
+
+    cache_paths = [CACHE, RENKU_TMP]
+    paths = [client.path / RENKU_HOME / p for p in cache_paths]
+
+    client.repository.clean(paths=paths)
diff --git a/renku/core/template/template.py b/renku/core/template/template.py
@@ -155,6 +155,7 @@ def copy_template_metadata_to_client():
         except OSError as e:
             # TODO: Use a general cleanup strategy: https://github.com/SwissDataScienceCenter/renku-python/issues/736
             if cleanup:
+                client.repository.reset(hard=True)
                 client.repository.clean()
 
             raise errors.TemplateUpdateError(f"Cannot write to '{destination}'") from e
diff --git a/renku/core/util/contexts.py b/renku/core/util/contexts.py
@@ -106,7 +106,7 @@ def measure(message="TOTAL"):
 
 def click_context(path, command):
     """Provide a click context with repo path injected."""
-    from renku.core.management import RENKU_HOME
+    from renku.core.constant import RENKU_HOME
     from renku.core.management.client import LocalClient
 
     return click.Context(
diff --git a/renku/core/util/git.py b/renku/core/util/git.py
@@ -137,7 +137,7 @@ def get_cache_directory_for_repository(client, url) -> Path:
         Path: The path of the cache.
 
     """
-    from renku.core.dataset.constant import CACHE
+    from renku.core.constant import CACHE
 
     return client.renku_path / CACHE / get_full_repository_path(url)
 
diff --git a/renku/core/util/metadata.py b/renku/core/util/metadata.py
@@ -84,8 +84,8 @@ def construct_creator(creator: Union[dict, str], ignore_email) -> Tuple[Optional
 
 def is_external_file(path: Union[Path, str], client_path: Path):
     """Checks if a path is an external file."""
+    from renku.core.constant import RENKU_HOME
     from renku.core.dataset.constant import POINTERS
-    from renku.core.management import RENKU_HOME
 
     path = client_path / path
     if not path.is_symlink() or not is_subpath(path=path, base=client_path):
diff --git a/renku/core/workflow/plan_factory.py b/renku/core/workflow/plan_factory.py
@@ -31,9 +31,9 @@
 
 from renku.command.command_builder.command import inject
 from renku.core import errors
+from renku.core.constant import RENKU_HOME, RENKU_TMP
 from renku.core.interface.client_dispatcher import IClientDispatcher
 from renku.core.interface.project_gateway import IProjectGateway
-from renku.core.management import RENKU_HOME
 from renku.core.util.git import is_path_safe
 from renku.core.util.metadata import is_external_file
 from renku.core.util.os import get_absolute_path, get_relative_path, is_subpath
@@ -50,8 +50,6 @@
 
 STARTED_AT = int(time.time() * 1000)
 
-RENKU_TMP = "tmp"
-
 
 class PlanFactory:
     """Factory for creating a plan from a command line call."""
diff --git a/renku/core/workflow/providers/toil.py b/renku/core/workflow/providers/toil.py
@@ -37,11 +37,10 @@
 
 from renku.command.echo import progressbar
 from renku.core import errors
+from renku.core.constant import RENKU_HOME, RENKU_TMP
 from renku.core.errors import WorkflowExecuteError
-from renku.core.management.config import RENKU_HOME
 from renku.core.plugin import hookimpl
 from renku.core.plugin.provider import RENKU_ENV_PREFIX
-from renku.core.workflow.plan_factory import RENKU_TMP
 from renku.domain_model.workflow.parameter import CommandParameterBase
 from renku.domain_model.workflow.plan import Plan
 from renku.domain_model.workflow.provider import IWorkflowProvider
diff --git a/renku/domain_model/template.py b/renku/domain_model/template.py
@@ -29,7 +29,7 @@
 import yaml
 
 from renku.core import errors
-from renku.core.management import RENKU_HOME
+from renku.core.constant import RENKU_HOME
 from renku.core.util.os import get_safe_relative_path, hash_file
 
 TEMPLATE_MANIFEST = "manifest.yaml"
diff --git a/renku/infrastructure/repository.py b/renku/infrastructure/repository.py
@@ -225,10 +225,9 @@ def checkout(self, reference: Union["Branch", "Tag", str]):
         """Check-out a specific reference."""
         self.run_git_command("checkout", reference)
 
-    def clean(self):
-        """Remove all untracked files and reset the repo."""
-        self.reset(hard=True)
-        self.run_git_command("clean", "-xdff")
+    def clean(self, paths: List[Union[Path, str]] = None):
+        """Remove untracked files."""
+        self.run_git_command("clean", "-xdff", paths)
 
     def fetch(
         self,
diff --git a/renku/ui/cli/__init__.py b/renku/ui/cli/__init__.py
diff --git a/renku/ui/cli/gc.py b/renku/ui/cli/gc.py
diff --git a/renku/ui/service/controllers/api/mixins.py b/renku/ui/service/controllers/api/mixins.py
diff --git a/tests/cli/test_datasets.py b/tests/cli/test_datasets.py
diff --git a/tests/cli/test_errors.py b/tests/cli/test_errors.py
diff --git a/tests/cli/test_gc.py b/tests/cli/test_gc.py
diff --git a/tests/cli/test_gitignore.py b/tests/cli/test_gitignore.py
diff --git a/tests/cli/test_migrate.py b/tests/cli/test_migrate.py

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`		`-8f835f7c5c76b60f8cd9785fd1e0cf31 cheatsheet.tex`
	`1`	`+3e7d7567c17eba64945b70a3c1009343 cheatsheet.tex`
`2`	`2`	`c70c179e07f04186ec05497564165f11 sdsc_cheatsheet.cls`