feat(core): v10 migrations (#3236)

Author: Panaetius

helm-chart/renku-core/values.yaml

@@ -91,6 +91,15 @@ sentry:
 # versions is the list of different deployment that support different metadata versions.
 versions:
   latest:
+    name: v10
+    prefix: "10"
+    nameOverride: ""
+    fullnameOverride: ""
+    image:
+      repository: renku/renku-core
+      tag: "v2.0.0"
+      pullPolicy: IfNotPresent
+  v9:
     name: v9
     prefix: "9"
     nameOverride: ""

renku/command/command_builder/database.py

@@ -20,7 +20,7 @@
 
 import json
 import os
-from typing import TYPE_CHECKING, Optional
+from typing import Optional
 
 from packaging.version import Version
 
@@ -40,22 +40,19 @@
 from renku.infrastructure.gateway.project_gateway import ProjectGateway
 from renku.infrastructure.storage.factory import StorageFactory
 
-if TYPE_CHECKING:
-    from renku.domain_model.project import Project
-
 
 class DatabaseCommand(Command):
     """Builder to get a database connection."""
 
     PRE_ORDER = 4
     POST_ORDER = 5
 
-    def __init__(self, builder: Command, write: bool = False, path: str = None, create: bool = False) -> None:
+    def __init__(self, builder: Command, write: bool = False, path: Optional[str] = None, create: bool = False) -> None:
         self._builder = builder
         self._write = write
         self._path = path
         self._create = create
-        self.project: Optional["Project"] = None
+        self.project_found: bool = False
 
     def _injection_pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
         """Create a Database singleton."""
@@ -80,8 +77,9 @@ def _injection_pre_hook(self, builder: Command, context: dict, *args, **kwargs)
             return
 
         try:
-            self.project = project_gateway.get_project()
-            minimum_renku_version = Version(self.project.minimum_renku_version)
+            project = project_gateway.get_project()
+            minimum_renku_version = Version(project.minimum_renku_version)
+            self.project_found = True
         except (KeyError, ImportError, ValueError):
             try:
                 with open(project_context.database_path / "project", "r") as f:
@@ -102,13 +100,14 @@ def _injection_pre_hook(self, builder: Command, context: dict, *args, **kwargs)
     def _post_hook(self, builder: Command, context: dict, result: CommandResult, *args, **kwargs) -> None:
         from renku.domain_model.project import Project
 
-        if (
-            self._write
-            and self.project is not None
-            and Version(self.project.minimum_renku_version) < Version(Project.minimum_renku_version)
-        ):
-            # NOTE: update minimum renku version on write as migrations might happen on the fly
-            self.project.minimum_renku_version = Project.minimum_renku_version
+        if self._write and self.project_found:
+            # NOTE: Fetch project again in case it was updated (the current reference would be put of date)
+            project_gateway = ProjectGateway()
+            project = project_gateway.get_project()
+
+            if Version(project.minimum_renku_version) < Version(Project.minimum_renku_version):
+                # NOTE: update minimum renku version on write as migrations might happen on the fly
+                project.minimum_renku_version = Project.minimum_renku_version
 
         project_context.pop_context()
 

renku/command/migrate.py

@@ -92,10 +92,10 @@ def _template_migration_check():
 
     try:
         project = project_context.project
-        template_source = project.template_source
-        template_ref = project.template_ref
-        template_id = project.template_id
-    except ValueError:
+        template_source = project.template_metadata.template_source
+        template_ref = project.template_metadata.template_ref
+        template_id = project.template_metadata.template_id
+    except (ValueError, AttributeError):
         project = None
         template_source = None
         template_ref = None
@@ -188,6 +188,10 @@ def _check_project():
         _ = project_context.project
     except ValueError:
         return MIGRATION_REQUIRED
+    else:
+        if hasattr(project_context.project, "template_source"):
+            # NOTE: v10 migration not done
+            return MIGRATION_REQUIRED
 
     # NOTE: ``project.automated_update`` is deprecated and we always allow template update for a project
     status = AUTOMATED_TEMPLATE_UPDATE_SUPPORTED
@@ -213,7 +217,7 @@ def _check_immutable_template_files(paths: List[str]):
     Returns:
         List of immutable template files.
     """
-    immutable_template_files = project_context.project.immutable_template_files or []
+    immutable_template_files = project_context.project.template_metadata.immutable_template_files or []
 
     return [p for p in paths if str(p) in immutable_template_files]
 

renku/command/rollback.py

@@ -266,14 +266,14 @@ def _get_modification_type_from_db(path: str) -> Optional[Tuple[str, str, str, d
             derived = database.get_by_id(db_object.derived_from)
             if db_object.name == derived.name:
                 change_type = "modified"
-        if db_object.invalidated_at:
+        if db_object.date_removed:
             change_type = "restored"
 
         return (
             f"Plan: {db_object.name}",
             change_type,
             f"plan_{db_object.name}",
-            db_object.invalidated_at or db_object.date_created,
+            db_object.date_removed or db_object.date_created,
         )
     elif isinstance(db_object, Dataset):
         change_type = "removed"

renku/command/schema/activity.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 """Activity JSON-LD schema."""
 
-from marshmallow import EXCLUDE, pre_dump
+from marshmallow import EXCLUDE
 
 from renku.command.schema.agent import PersonSchema, SoftwareAgentSchema
 from renku.command.schema.annotation import AnnotationSchema
@@ -35,29 +35,6 @@
 from renku.domain_model.provenance.parameter import ParameterValue
 
 
-class _ObjectWrapper:
-    """Object wrapper that allows temporarily overriding fields of immutable objects."""
-
-    def __init__(self, wrapped, **override):
-        self.__wrapped = wrapped
-        self.__override = override
-
-    def __getattr__(self, name):
-        if name in self.__override:
-            return self.__override[name]
-
-        return getattr(self.__wrapped, name)
-
-
-def _fix_id(obj):
-    """Fix ids under an activity that were wrong due to a bug."""
-
-    if not obj.id.startswith("/activities/") and not obj.id.startswith("/workflow-file-activity-collection/"):
-        obj = _ObjectWrapper(obj, id=f"/activities/{obj.id}")
-
-    return obj
-
-
 class AssociationSchema(JsonLDSchema):
     """Association schema."""
 
@@ -72,11 +49,6 @@ class Meta:
     id = fields.Id()
     plan = Nested(prov.hadPlan, [PlanSchema, WorkflowFilePlanSchema, WorkflowFileCompositePlanSchema])
 
-    @pre_dump
-    def _pre_dump(self, obj, **kwargs):
-        """Pre-dump hook."""
-        return _fix_id(obj)
-
 
 class UsageSchema(JsonLDSchema):
     """Usage schema."""
@@ -92,11 +64,6 @@ class Meta:
     # TODO: DatasetSchema, DatasetFileSchema
     entity = Nested(prov.entity, [EntitySchema, CollectionSchema])
 
-    @pre_dump
-    def _pre_dump(self, obj, **kwargs):
-        """Pre-dump hook."""
-        return _fix_id(obj)
-
 
 class GenerationSchema(JsonLDSchema):
     """Generation schema."""
@@ -112,11 +79,6 @@ class Meta:
     # TODO: DatasetSchema, DatasetFileSchema
     entity = Nested(prov.qualifiedGeneration, [EntitySchema, CollectionSchema], reverse=True)
 
-    @pre_dump
-    def _pre_dump(self, obj, **kwargs):
-        """Pre-dump hook."""
-        return _fix_id(obj)
-
 
 class ParameterValueSchema(JsonLDSchema):
     """ParameterValue schema."""
@@ -133,11 +95,6 @@ class Meta:
     parameter = fields.IRI(schema.valueReference, attribute="parameter_id")
     value = fields.Raw(schema.value)
 
-    @pre_dump
-    def _pre_dump(self, obj, **kwargs):
-        """Pre-dump hook."""
-        return _fix_id(obj)
-
 
 class ActivitySchema(JsonLDSchema):
     """Activity schema."""
@@ -167,11 +124,6 @@ class Meta:
     started_at_time = fields.DateTime(prov.startedAtTime, add_value_types=True)
     usages = Nested(prov.qualifiedUsage, UsageSchema, many=True)
 
-    @pre_dump
-    def _pre_dump(self, obj, **kwargs):
-        """Pre-dump hook."""
-        return _fix_id(obj)
-
 
 class WorkflowFileActivityCollectionSchema(JsonLDSchema):
     """WorkflowFileActivityCollection schema."""

renku/command/schema/composite_plan.py

@@ -41,7 +41,8 @@ class Meta:
     creators = Nested(schema.creator, PersonSchema, many=True)
     mappings = Nested(renku.hasMappings, [ParameterMappingSchema], many=True, load_default=None)
     date_created = fields.DateTime(schema.dateCreated, format="iso")
-    invalidated_at = fields.DateTime(prov.invalidatedAtTime, format="iso")
+    date_modified = fields.DateTime(schema.dateModified, format="iso")
+    date_removed = fields.DateTime(prov.invalidatedAtTime, format="iso")
     keywords = fields.List(schema.keywords, fields.String(), load_default=None)
     name = fields.String(schema.name, load_default=None)
     derived_from = fields.IRI(prov.wasDerivedFrom, load_default=None)

renku/command/schema/dataset.py

@@ -148,6 +148,7 @@ class Meta:
     annotations = Nested(oa.hasTarget, AnnotationSchema, reverse=True, many=True)
     creators = Nested(schema.creator, PersonSchema, many=True)
     date_created = fields.DateTime(schema.dateCreated, load_default=None, format="iso", extra_formats=("%Y-%m-%d",))
+    date_modified = fields.DateTime(schema.dateModified, load_default=None, format="iso", extra_formats=("%Y-%m-%d",))
     date_removed = fields.DateTime(prov.invalidatedAtTime, load_default=None, format="iso")
     date_published = fields.DateTime(
         schema.datePublished, load_default=None, format="%Y-%m-%d", extra_formats=("iso", "%Y-%m-%dT%H:%M:%S")

renku/command/schema/plan.py

@@ -17,8 +17,6 @@
 # limitations under the License.
 """Represent run templates."""
 
-from datetime import timezone
-
 import marshmallow
 
 from renku.command.schema.agent import PersonSchema
@@ -46,7 +44,8 @@ class Meta:
     id = fields.Id()
     inputs = Nested(renku.hasInputs, CommandInputSchema, many=True, load_default=None)
     date_created = fields.DateTime(schema.dateCreated, format="iso")
-    invalidated_at = fields.DateTime(prov.invalidatedAtTime, format="iso")
+    date_modified = fields.DateTime(schema.dateModified, format="iso")
+    date_removed = fields.DateTime(prov.invalidatedAtTime, format="iso")
     keywords = fields.List(schema.keywords, fields.String(), load_default=None)
     name = fields.String(schema.name, load_default=None)
     derived_from = fields.IRI(prov.wasDerivedFrom, load_default=None)
@@ -55,15 +54,3 @@ class Meta:
     parameters = Nested(renku.hasArguments, CommandParameterSchema, many=True, load_default=None)
     success_codes = fields.List(renku.successCodes, fields.Integer(), load_default=[0])
     annotations = Nested(oa.hasTarget, AnnotationSchema, reverse=True, many=True)
-
-    @marshmallow.pre_dump
-    def _pre_dump(self, in_data, **kwargs):
-        """Fix data on dumping."""
-        if in_data.invalidated_at is not None and in_data.invalidated_at.tzinfo is None:
-            # NOTE: There was a bug that caused invalidated_at to be set without timezone (as UTC time)
-            # so we patch in the timezone here
-            in_data.unfreeze()
-            in_data.invalidated_at = in_data.invalidated_at.replace(microsecond=0).astimezone(timezone.utc)
-            in_data.freeze()
-
-        return in_data

renku/command/schema/project.py

@@ -37,17 +37,25 @@ class Meta:
 
     agent_version = StringList(schema.agent, load_default="pre-0.11.0")
     annotations = Nested(oa.hasTarget, AnnotationSchema, reverse=True, many=True)
-    automated_update = fields.Boolean(renku.automatedTemplateUpdate, load_default=True)
     creator = Nested(schema.creator, PersonSchema, load_default=None)
     date_created = DateTimeList(schema.dateCreated, load_default=None, format="iso", extra_formats=("%Y-%m-%d",))
     description = fields.String(schema.description, load_default=None)
     id = fields.Id(load_default=None)
-    immutable_template_files = fields.List(renku.immutableTemplateFiles, fields.String(), load_default=list())
+    immutable_template_files = fields.List(
+        renku.immutableTemplateFiles,
+        fields.String(),
+        load_default=list(),
+        attribute="template_metadata.immutable_template_files",
+    )
     name = fields.String(schema.name, load_default=None)
-    template_id = fields.String(renku.templateId, load_default=None)
-    template_metadata = fields.String(renku.templateMetadata, load_default=None)
-    template_ref = fields.String(renku.templateReference, load_default=None)
-    template_source = fields.String(renku.templateSource, load_default=None)
-    template_version = fields.String(renku.templateVersion, load_default=None)
+    template_id = fields.String(renku.templateId, load_default=None, attribute="template_metadata.template_id")
+    template_metadata = fields.String(renku.templateMetadata, load_default=None, attribute="template_metadata.metadata")
+    template_ref = fields.String(renku.templateReference, load_default=None, attribute="template_metadata.template_ref")
+    template_source = fields.String(
+        renku.templateSource, load_default=None, attribute="template_metadata.template_source"
+    )
+    template_version = fields.String(
+        renku.templateVersion, load_default=None, attribute="template_metadata.template_version"
+    )
     version = StringList(schema.schemaVersion, load_default="1")
     keywords = fields.List(schema.keywords, fields.String(), load_default=None)

renku/command/view_model/log.py

@@ -197,10 +197,10 @@ def from_dataset(cls, dataset: "Dataset") -> "DatasetLogViewModel":
             descriptions.append("deleted")
             details.deleted = True
 
-        previous_dataset = None
+        previous_dataset: Optional[Dataset] = None
 
         if dataset.is_derivation():
-            previous_dataset = dataset_gateway.get_by_id(dataset.derived_from.url_id)  # type: ignore
+            previous_dataset = dataset_gateway.get_by_id(dataset.derived_from.value)  # type: ignore
 
         current_files = {f for f in dataset.dataset_files if not f.date_removed}
         previous_files = set()
@@ -222,7 +222,7 @@ def from_dataset(cls, dataset: "Dataset") -> "DatasetLogViewModel":
 
             descriptions.append(f"{len(new_files)} file(s) added")
             details.files_added = [str(f.entity.path) for f in new_files]
-            details.modified = True
+            details.modified = bool(previous_files)
 
         if previous_files and {f.id for f in previous_files}.difference({f.id for f in current_files}):
             # NOTE: Files removed
@@ -277,8 +277,8 @@ def from_dataset(cls, dataset: "Dataset") -> "DatasetLogViewModel":
                 details.keywords_removed = list(previous_keywords.difference(current_keywords))
                 modified = True
 
-            current_images = set(dataset.images)
-            previous_images = set(previous_dataset.images)
+            current_images = set(dataset.images) if dataset.images else set()
+            previous_images = set(previous_dataset.images) if previous_dataset.images else set()
 
             if current_images != previous_images:
                 details.images_changed_to = [i.content_url for i in current_images]