Merge branch 'develop' into release/v2.7.0

Author: olevski

CHANGES.rst

@@ -18,7 +18,7 @@
 Changes
 =======
 
-`2.7.0 <https://github.com/SwissDataScienceCenter/renku-python/compare/v2.6.1...v2.7.0>`__ (2023-08-09)
+`2.7.0 <https://github.com/SwissDataScienceCenter/renku-python/compare/v2.6.2...v2.7.0>`__ (2023-08-09)
 -------------------------------------------------------------------------------------------------------
 
 Bug Fixes
@@ -48,6 +48,41 @@ Features
    (`#3534 <https://github.com/SwissDataScienceCenter/renku-python/issues/3534>`__)
    (`a9994a8 <https://github.com/SwissDataScienceCenter/renku-python/commit/a9994a8cb2541527ca9be731037cc60b03d62cea>`__)
 
+`2.6.2 <https://github.com/SwissDataScienceCenter/renku-python/compare/v2.6.1...v2.6.2>`__ (2023-08-23)
+-------------------------------------------------------------------------------------------------------
+
+Bug Fixes
+~~~~~~~~~
+
+-  **cli:** correct bad plan ID format
+   (`#3594 <https://github.com/SwissDataScienceCenter/renku-python/issues/3594>`__)
+   (`c418c17 <https://github.com/SwissDataScienceCenter/renku-python/commit/c418c178d03a5caac126d14cc089064ee13f2747>`__)
+-  **cli:** fix overwriting dataset tags
+   (`#3497 <https://github.com/SwissDataScienceCenter/renku-python/issues/3497>`__)
+   (`ef6d214 <https://github.com/SwissDataScienceCenter/renku-python/commit/ef6d214e165ba877b5acc26427d5663366aaaa29>`__)
+-  **cli:** parse security_opt as list not tuple so that session launches from the CLI work
+   (`#3587 <https://github.com/SwissDataScienceCenter/renku-python/issues/3587>`__)
+   (`d74d35f <https://github.com/SwissDataScienceCenter/renku-python/commit/d74d35ff3428118945a1df1b2f52eebcfcf31132>`__)
+-  **core:** properly formatted dataset image IDs
+   (`#3596 <https://github.com/SwissDataScienceCenter/renku-python/issues/3596>`__)
+   (`f624b2b <https://github.com/SwissDataScienceCenter/renku-python/commit/f624b2bf261d97b07c88243f674f544613753e28>`__)
+-  **service:** speed up cache.migration_check endpoint
+   (`#3597 <https://github.com/SwissDataScienceCenter/renku-python/issues/3597>`__)
+   (`20b5589 <https://github.com/SwissDataScienceCenter/renku-python/commit/20b5589ea2639b4ff017fc390a9b685842c9685d>`__)
+-  **service:** serialize project versions as integer
+   (`#3591 <https://github.com/SwissDataScienceCenter/renku-python/issues/3591>`__)
+   (`5624573 <https://github.com/SwissDataScienceCenter/renku-python/commit/5624573160c2820770093f8752d1975a18e785e6>`__)
+
+`2.6.1 <https://github.com/SwissDataScienceCenter/renku-python/compare/v2.6.1...v2.6.2>`__ (2023-07-13)
+-------------------------------------------------------------------------------------------------------
+
+Bug Fixes
+~~~~~~~~~
+
+- **service:** fixes issues with the scaling of the core service
+  (`#3555 <https://github.com/SwissDataScienceCenter/renku-python/pull/3555>`__)
+  (`4288ada <https://github.com/SwissDataScienceCenter/renku-python/commit/4288ada0b4ec658089b25d12add8a1d16955498a>`__)
+
 `2.6.0 <https://github.com/SwissDataScienceCenter/renku-python/compare/v2.5.0...v2.6.0>`__ (2023-06-20)
 -------------------------------------------------------------------------------------------------------
 

poetry.lock

@@ -83,7 +83,7 @@ psutil = ">=5.4.7,<5.9.2"
 pydantic = "==1.10.7"
 pyjwt = ">=2.1.0,<2.5.0"
 pyld = "==2.0.3"
-pyshacl = ">=0.17.2,<0.19.2"
+pyshacl = ">=0.20.0,<0.24.0"
 python-dateutil = "<2.8.3,>=2.6.1"
 python-gitlab = ">=2.10.1,<3.8.2"
 pyyaml = "<6.1.0,>=5.4"
@@ -149,7 +149,7 @@ pytest-flake8 = ">=1.0.6,<1.1.2"
 pytest-lazy-fixture = ">=0.6.3,<0.7.0"
 pytest-mock = ">=3.2.0,<3.11.0"
 pytest-pep8 = "==1.0.6"
-pytest-recording = "==0.12.2"
+pytest-recording = "==0.13.0"
 pytest-timeout = "==2.1.0"
 pytest-xdist = ">=1.34.0,<3.4.0"
 responses = ">=0.22.0,<0.24.0"
@@ -342,5 +342,5 @@ exclude = ["docs"]
 
 
 [build-system]
-requires = ["poetry-core>=1.3.0<1.7.0", "poetry-dynamic-versioning==0.21.5", "gitpython==3.1.24"]
+requires = ["poetry-core>=1.3.0,<1.7.0", "poetry-dynamic-versioning==0.21.5", "gitpython==3.1.24"]
 build-backend = "poetry_dynamic_versioning.backend"

pyproject.toml

@@ -28,7 +28,7 @@
 from .project import check_project_id_group
 from .storage import check_lfs_info
 from .validate_shacl import check_datasets_structure, check_project_structure
-from .workflow import check_activity_catalog, check_plan_modification_date
+from .workflow import check_activity_catalog, check_plan_id, check_plan_modification_date
 
 # Checks will be executed in the order as they are listed in __all__. They are mostly used in ``doctor`` command to
 # inspect broken things. The order of operations matters when fixing issues, so, don't sort this list.
@@ -47,5 +47,6 @@
     "check_project_id_group",
     "check_project_structure",
     "check_plan_modification_date",
+    "check_plan_id",
     "check_activity_dates",
 )

renku/command/checks/__init__.py

@@ -127,3 +127,44 @@ def fix_plan_dates(plans: List[AbstractPlan], plan_gateway):
             if plan.date_removed and plan.date_removed < plan.date_created:
                 plan.date_removed = plan.date_created + timedelta(seconds=1)
             plan.freeze()
+
+
+@inject.autoparams("plan_gateway")
+def check_plan_id(fix, plan_gateway: IPlanGateway, **_) -> Tuple[bool, bool, Optional[str]]:
+    """Check if all plans have correct IDs.
+
+    Args:
+        fix(bool): Whether to fix found issues.
+        plan_gateway(IPlanGateway): Injected PlanGateway.
+        _: keyword arguments.
+
+    Returns:
+        Tuple[bool, Optional[str]]: Tuple of whether there are plans with invalid IDs, if an automated fix is
+            available and a string of their IDs
+    """
+    plans: List[AbstractPlan] = plan_gateway.get_all_plans()
+
+    to_be_processed = []
+    for plan in plans:
+        if isinstance(plan.id, str) and plan.id.startswith("/plans//plans"):
+            to_be_processed.append(plan)
+
+    if not to_be_processed:
+        return True, False, None
+    if not fix:
+        ids = [plan.id for plan in to_be_processed]
+        message = (
+            WARNING
+            + "The following workflows have incorrect IDs (use 'renku doctor --fix' to fix them):\n\t"
+            + "\n\t".join(ids)
+        )
+        return False, True, message
+
+    for plan in to_be_processed:
+        plan.unfreeze()
+        plan.id = plan.id.replace("//plans/", "/")
+        plan.freeze()
+    project_context.database.commit()
+    communication.info("Workflow IDs were fixed")
+
+    return True, False, None

renku/command/checks/workflow.py

@@ -95,12 +95,10 @@ def _update_config(
     """
     section, section_key = _split_section_and_key(key)
     if remove:
-        value = remove_value(section, section_key, global_only=global_only)
-        if value is None:
-            raise errors.ParameterError(f'Key "{key}" not found.')
+        remove_value(section, section_key, global_only=global_only)
     else:
         set_value(section, section_key, value, global_only=global_only)
-        return value
+    return value
 
 
 def update_config():

renku/command/config.py

@@ -39,7 +39,7 @@ def search_session_providers_command():
 
 def session_list_command():
     """List all the running interactive sessions."""
-    return Command().command(session_list)
+    return Command().command(session_list).with_database(write=False)
 
 
 def session_start_command():

renku/command/session.py

@@ -1390,11 +1390,6 @@ def full_path(self):
         path = project_context.path / self.path
         return Path(os.path.abspath(path))
 
-    @property
-    def filesize(self):
-        """Return file size."""
-        return None if self.filesize is None else self.filesize
-
     def __attrs_post_init__(self):
         """Set the property "name" after initialization."""
         super().__attrs_post_init__()

renku/core/migration/models/v9.py

@@ -299,8 +299,6 @@ def session_list(self, project_name: str, ssh_garbage_collection: bool = True) -
             params=self._get_renku_project_name_parts(),
         )
         if sessions_res.status_code == 200:
-            system_config = SystemSSHConfig()
-            name = self._project_name_from_full_project_name(project_name)
             sessions = [
                 Session(
                     id=session["name"],
@@ -310,7 +308,8 @@ def session_list(self, project_name: str, ssh_garbage_collection: bool = True) -
                     commit=session.get("annotations", {}).get("renku.io/commit-sha"),
                     branch=session.get("annotations", {}).get("renku.io/branch"),
                     provider="renkulab",
-                    ssh_enabled=system_config.session_config_path(name, session["name"]).exists(),
+                    ssh_enabled=get_value("renku", "ssh_supported") == "true"
+                    or project_context.project.template_metadata.ssh_supported,
                 )
                 for session in sessions_res.json().get("servers", {}).values()
             ]

renku/core/session/renkulab.py

@@ -142,6 +142,11 @@ def session_start(
     """
     from renku.domain_model.project_context import project_context
 
+    # NOTE: The Docker client in Python requires the parameters below to be a list and will fail with a tuple.
+    # Click will convert parameters with the flag "many" set to True to tuples.
+    kwargs["security_opt"] = list(kwargs.get("security_opt", []))
+    kwargs["device_cgroup_rules"] = list(kwargs.get("device_cgroup_rules", []))
+
     pinned_image = get_value("interactive", "image")
     if pinned_image and image_name is None:
         image_name = pinned_image
@@ -348,3 +353,8 @@ def ssh_setup(existing_key: Optional[Path] = None, force: bool = False):
             """
         )
         f.write(content)
+
+    communication.warn(
+        "This command does not add any public SSH keys to your project. "
+        "Keys have to be added manually or by using the 'renku session start' command with the '--ssh' flag."
+    )