Add health check that MySQL Router is part of InnoDB cluster (#59)

Ported from
canonical/mysql-router-k8s-operator#116

Author: carlcsaposs-canonical

src/abstract_charm.py

@@ -34,8 +34,11 @@ def __init__(self, *args) -> None:
         self._authenticated_workload_type = workload.AuthenticatedWorkload
         self._database_requires = relations.database_requires.RelationEndpoint(self)
         self._database_provides = relations.database_provides.RelationEndpoint(self)
-        self.framework.observe(self.on.start, self._on_start)
-        self.framework.observe(self.on.leader_elected, self._on_leader_elected)
+        self.framework.observe(self.on.update_status, self.reconcile_database_relations)
+        # Set status on first start if no relations active
+        self.framework.observe(self.on.start, self.reconcile_database_relations)
+        # Update app status
+        self.framework.observe(self.on.leader_elected, self.reconcile_database_relations)
 
     @property
     @abc.abstractmethod
@@ -107,8 +110,8 @@ def _determine_app_status(self, *, event) -> ops.StatusBase:
     def _determine_unit_status(self, *, event) -> ops.StatusBase:
         """Report unit status."""
         statuses = []
-        if not self.get_workload(event=event).container_ready:
-            statuses.append(ops.MaintenanceStatus("Waiting for container"))
+        workload_ = self.get_workload(event=event)
+        statuses.append(workload_.get_status(event))
         return self._prioritize_statuses(statuses)
 
     def set_status(self, *, event) -> None:
@@ -173,11 +176,3 @@ def reconcile_database_relations(self, event=None) -> None:
         elif workload_.container_ready:
             workload_.disable()
         self.set_status(event=event)
-
-    def _on_start(self, _) -> None:
-        # Set status on first start if no relations active
-        self.set_status(event=None)
-
-    def _on_leader_elected(self, _) -> None:
-        # Update app status
-        self.set_status(event=None)

src/mysql_shell.py

@@ -169,3 +169,16 @@ def delete_user(self, username: str) -> None:
         logger.debug(f"Deleting {username=}")
         self._run_sql([f"DROP USER `{username}`"])
         logger.debug(f"Deleted {username=}")
+
+    def is_router_in_cluster_set(self, router_id: str) -> bool:
+        """Check if MySQL Router is part of InnoDB ClusterSet."""
+        logger.debug(f"Checking if {router_id=} in cluster set")
+        output = json.loads(
+            self._run_commands(
+                ["cluster_set = dba.get_cluster_set()", "print(cluster_set.list_routers())"]
+            )
+        )
+        cluster_set_router_ids = output["routers"].keys()
+        logger.debug(f"{cluster_set_router_ids=}")
+        logger.debug(f"Checked if {router_id in cluster_set_router_ids=}")
+        return router_id in cluster_set_router_ids

src/workload.py

@@ -10,6 +10,8 @@
 import string
 import typing
 
+import ops
+
 import container
 import mysql_shell
 
@@ -92,6 +94,11 @@ def disable_tls(self) -> None:
             file.unlink(missing_ok=True)
         logger.debug("Disabled TLS")
 
+    def get_status(self, event) -> typing.Optional[ops.StatusBase]:
+        """Report non-active status."""
+        if not self.container_ready:
+            return ops.MaintenanceStatus("Waiting for container")
+
 
 class AuthenticatedWorkload(Workload):
     """Workload with connection to MySQL cluster"""
@@ -122,7 +129,7 @@ def shell(self) -> mysql_shell.Shell:
     def _router_id(self) -> str:
         """MySQL Router ID in InnoDB Cluster metadata
 
-        Used to remove MySQL Router metadata from InnoDB cluster
+        Used to remove MySQL Router metadata from InnoDB Cluster
         """
         # MySQL Router is bootstrapped without `--directory`—there is one system-wide instance.
         return f"{socket.getfqdn()}::system"
@@ -235,3 +242,16 @@ def disable_tls(self) -> None:
         super().disable_tls()
         if self._container.mysql_router_service_enabled:
             self._restart(tls=False)
+
+    def get_status(self, event) -> typing.Optional[ops.StatusBase]:
+        """Report non-active status."""
+        if status := super().get_status(event):
+            return status
+        if not self.shell.is_router_in_cluster_set(self._router_id):
+            # Router should not be removed from ClusterSet after bootstrap (except by MySQL charm
+            # when MySQL Router unit departs relation).
+            # If Router is not part of ClusterSet after bootstrap, it most likely was manually
+            # removed.
+            return ops.BlockedStatus(
+                "Router was manually removed from MySQL ClusterSet. Remove & re-deploy unit"
+            )

tests/unit/conftest.py

@@ -35,6 +35,7 @@ def patch(monkeypatch):
     monkeypatch.setattr(
         "workload.AuthenticatedWorkload._router_username", lambda *args, **kwargs: ""
     )
+    monkeypatch.setattr("mysql_shell.Shell.is_router_in_cluster_set", lambda *args, **kwargs: True)
 
 
 @pytest.fixture(autouse=True)