Add in-place upgrades (#138)

Based on specification: [DA058 - In-Place Upgrades - Kubernetes
v2](https://docs.google.com/document/d/1tLjknwHudjcHs42nzPVBNkHs98XxAOT2BXGGpP7NyEU/edit)

Author: carlcsaposs-canonical

.github/workflows/ci.yaml

@@ -100,6 +100,13 @@ jobs:
         run: |
           pipx install tox
           pipx install poetry
+      - name: Free up disk space
+        run: |
+          # From https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - name: Setup operator environment
         uses: charmed-kubernetes/actions-operator@main
         with:

CONTRIBUTING.md

@@ -66,8 +66,9 @@ juju add-model dev
 # Enable DEBUG logging
 juju model-config logging-config="<root>=INFO;unit=DEBUG"
 # Deploy the charm
+# `--trust` needed if Role Based Access Control (RBAC) (https://kubernetes.io/docs/concepts/security/rbac-good-practices/) is enabled on Kubernetes
 juju deploy ./mysqlrouter-operator_ubuntu-20.04-amd64.charm \
-    --resource mysql-router-image=mysql/mysql-router:8.0
+    --resource mysql-router-image=mysql/mysql-router:8.0 --trust
 ```
 
 ## Canonical Contributor Agreement

README.md

@@ -14,12 +14,12 @@ flowchart TD
 
 ## Usage
 
-This charm must be used coupled with mysql-operator charm, through a relation, e.g.:
+This charm must be used coupled with mysql charm, through a relation, e.g.:
 
 ```bash
-juju deploy mysql-operator
-juju deploy mysql-router-operator
-juju add-relation mysql-operator mysql-router-operator
+juju deploy mysql-k8s --trust
+juju deploy mysql-router-k8s --trust
+juju integrate mysql-k8s mysql-router-k8s
 ```
 
 ## Relations

actions.yaml

@@ -1,6 +1,13 @@
 # Copyright 2023 Canonical Ltd.
 # See LICENSE file for licensing details.
 
+resume-upgrade:
+  description: Upgrade remaining units (after you manually verified that upgraded units are healthy).
+  params:
+    force:
+      type: boolean
+      description: Force upgrade of next unit if an upgraded unit has non-active status.
+  required: []
 set-tls-private-key:
   description:
     Set the private key, which will be used for certificate signing requests (CSR). Run

charm_version

@@ -0,0 +1 @@
+1

charmcraft.yaml

@@ -19,6 +19,9 @@ parts:
           exit 1
       fi
     charm-entrypoint: src/kubernetes_charm.py
+    prime:
+      - charm_version
+      - workload_version
     build-packages:
       - libffi-dev
       - libssl-dev

metadata.yaml

@@ -32,6 +32,11 @@ requires:
     interface: tls-certificates
     limit: 1
 peers:
+  upgrade-version-a:
+    # Relation versioning scheme:
+    # DA056 - Upgrading in-place upgrade protocol
+    # https://docs.google.com/document/d/1H7qy5SAwLiCOKO9xMQJbbQP5_-jGV6Lhi-mJOk4gZ08/edit
+    interface: upgrade
   mysql-router-peers:
     interface: mysql_router_peers
 resources:

poetry.lock

@@ -13,6 +13,7 @@ python = "^3.10"
 ops = "^2.6.0"
 lightkube = "^0.14.0"
 tenacity = "^8.2.3"
+poetry-core = "^1.7.0"
 
 [tool.poetry.group.charm-libs.dependencies]
 # data_platform_libs/v0/data_interfaces.py

pyproject.toml

@@ -15,6 +15,7 @@
 import lifecycle
 import relations.database_provides
 import relations.database_requires
+import upgrade
 import workload
 
 logger = logging.getLogger(__name__)
@@ -34,11 +35,21 @@ def __init__(self, *args) -> None:
         self._authenticated_workload_type = workload.AuthenticatedWorkload
         self._database_requires = relations.database_requires.RelationEndpoint(self)
         self._database_provides = relations.database_provides.RelationEndpoint(self)
-        self.framework.observe(self.on.update_status, self.reconcile_database_relations)
-        # Set status on first start if no relations active
-        self.framework.observe(self.on.start, self.reconcile_database_relations)
+        self.framework.observe(self.on.update_status, self.reconcile)
+        self.framework.observe(
+            self.on[upgrade.PEER_RELATION_ENDPOINT_NAME].relation_changed, self.reconcile
+        )
+        self.framework.observe(
+            self.on[upgrade.RESUME_ACTION_NAME].action, self._on_resume_upgrade_action
+        )
+        # (For Kubernetes) Reset partition after scale down
+        self.framework.observe(
+            self.on[upgrade.PEER_RELATION_ENDPOINT_NAME].relation_departed, self.reconcile
+        )
+        # Handle upgrade & set status on first start if no relations active
+        self.framework.observe(self.on.start, self.reconcile)
         # Update app status
-        self.framework.observe(self.on.leader_elected, self.reconcile_database_relations)
+        self.framework.observe(self.on.leader_elected, self.reconcile)
 
     @property
     @abc.abstractmethod
@@ -59,6 +70,11 @@ def _tls_certificate_saved(self) -> bool:
     def _container(self) -> container.Container:
         """Workload container (snap or ROCK)"""
 
+    @property
+    @abc.abstractmethod
+    def _upgrade(self) -> typing.Optional[upgrade.Upgrade]:
+        pass
+
     @property
     @abc.abstractmethod
     def _read_write_endpoint(self) -> str:
@@ -88,8 +104,8 @@ def _prioritize_statuses(statuses: typing.List[ops.StatusBase]) -> ops.StatusBas
         """
         status_priority = (
             ops.BlockedStatus,
-            ops.WaitingStatus,
             ops.MaintenanceStatus,
+            ops.WaitingStatus,
             # Catch any unknown status type
             ops.StatusBase,
         )
@@ -101,6 +117,11 @@ def _prioritize_statuses(statuses: typing.List[ops.StatusBase]) -> ops.StatusBas
 
     def _determine_app_status(self, *, event) -> ops.StatusBase:
         """Report app status."""
+        if self._upgrade and (upgrade_status := self._upgrade.app_status):
+            # Upgrade status should take priority over relation status—even if the status level is
+            # normally lower priority.
+            # (Relations should not be modified during upgrade.)
+            return upgrade_status
         statuses = []
         for endpoint in (self._database_requires, self._database_provides):
             if status := endpoint.get_status(event):
@@ -111,16 +132,19 @@ def _determine_unit_status(self, *, event) -> ops.StatusBase:
         """Report unit status."""
         statuses = []
         workload_ = self.get_workload(event=event)
-        statuses.append(workload_.get_status(event))
+        statuses.append(workload_.status)
+        if self._upgrade:
+            statuses.append(self._upgrade.unit_juju_status)
         return self._prioritize_statuses(statuses)
 
-    def set_status(self, *, event) -> None:
+    def set_status(self, *, event, app=True, unit=True) -> None:
         """Set charm status."""
-        if self._unit_lifecycle.authorized_leader:
+        if app and self._unit_lifecycle.authorized_leader:
             self.app.status = self._determine_app_status(event=event)
             logger.debug(f"Set app status to {self.app.status}")
-        self.unit.status = self._determine_unit_status(event=event)
-        logger.debug(f"Set unit status to {self.unit.status}")
+        if unit:
+            self.unit.status = self._determine_unit_status(event=event)
+            logger.debug(f"Set unit status to {self.unit.status}")
 
     def wait_until_mysql_router_ready(self) -> None:
         """Wait until a connection to MySQL Router is possible.
@@ -149,21 +173,38 @@ def wait_until_mysql_router_ready(self) -> None:
     #  Handlers
     # =======================
 
-    def reconcile_database_relations(self, event=None) -> None:
-        """Handle database requires/provides events."""
+    def reconcile(self, event=None) -> None:  # noqa: C901
+        """Handle most events."""
+        if not self._upgrade:
+            logger.debug("Peer relation not available")
+            return
+        if self._upgrade.unit_state == "restarting":
+            if not self._upgrade.is_compatible:
+                self.unit.status = ops.BlockedStatus(
+                    "Upgrade incompatible. Rollback to previous revision with `juju refresh`"
+                )
+                self.set_status(event=event, unit=False)
+                return
         workload_ = self.get_workload(event=event)
         logger.debug(
             "State of reconcile "
             f"{self._unit_lifecycle.authorized_leader=}, "
             f"{isinstance(workload_, workload.AuthenticatedWorkload)=}, "
             f"{workload_.container_ready=}, "
-            f"{self._database_requires.is_relation_breaking(event)=}"
+            f"{self._database_requires.is_relation_breaking(event)=}, "
+            f"{self._upgrade.in_progress=}"
         )
         if self._unit_lifecycle.authorized_leader:
             if self._database_requires.is_relation_breaking(event):
+                if self._upgrade.in_progress:
+                    logger.warning(
+                        "Modifying relations during an upgrade is not supported. The charm may be in a broken, unrecoverable state. Re-deploy the charm"
+                    )
                 self._database_provides.delete_all_databags()
             elif (
-                isinstance(workload_, workload.AuthenticatedWorkload) and workload_.container_ready
+                not self._upgrade.in_progress
+                and isinstance(workload_, workload.AuthenticatedWorkload)
+                and workload_.container_ready
             ):
                 self._database_provides.reconcile_users(
                     event=event,
@@ -175,4 +216,23 @@ def reconcile_database_relations(self, event=None) -> None:
             workload_.enable(tls=self._tls_certificate_saved, unit_name=self.unit.name)
         elif workload_.container_ready:
             workload_.disable()
+        if not workload_.status:
+            self._upgrade.unit_state = "healthy"
+        if self._unit_lifecycle.authorized_leader:
+            self._upgrade.reconcile_partition()
+            if not self._upgrade.in_progress:
+                self._upgrade.set_versions_in_app_databag()
         self.set_status(event=event)
+
+    def _on_resume_upgrade_action(self, event: ops.ActionEvent) -> None:
+        if not self._unit_lifecycle.authorized_leader:
+            message = f"Must run action on leader unit. (e.g. `juju run {self.app.name}/leader {upgrade.RESUME_ACTION_NAME}`)"
+            logger.debug(f"Resume upgrade event failed: {message}")
+            event.fail(message)
+            return
+        if not self._upgrade or not self._upgrade.in_progress:
+            message = "No upgrade in progress"
+            logger.debug(f"Resume upgrade event failed: {message}")
+            event.fail(message)
+            return
+        self._upgrade.reconcile_partition(action_event=event)