DPE-4200 Scale-up from zero (#414)

* (WIP) detect and treat scale up from zero

* scale up from zero

* bump outdated lib

* fix base data_keys

* postpone secondaries to after leader/primary

Author: paulomach

src/charm.py

@@ -261,6 +261,19 @@ def unit_address(self) -> str:
         """Return the address of this unit."""
         return self.get_unit_address()
 
+    @property
+    def is_new_unit(self) -> bool:
+        """Return whether the unit is a clean state.
+
+        e.g. scaling from zero units
+        """
+        _default_unit_data_keys = {
+            "egress-subnets",
+            "ingress-address",
+            "private-address",
+        }
+        return self.unit_peer_data.keys() == _default_unit_data_keys
+
     def get_unit_hostname(self, unit_name: Optional[str] = None) -> str:
         """Get the hostname.localdomain for a unit.
 
@@ -424,8 +437,6 @@ def _reconcile_pebble_layer(self, container: Container) -> None:
             ):
                 container.stop(MYSQLD_EXPORTER_SERVICE)
 
-            self._on_update_status(None)
-
     def _restart(self, event: EventBase) -> None:
         """Restart the service."""
         if self.peers.units != self.restart_peers.units:
@@ -702,6 +713,21 @@ def _on_mysql_pebble_ready(self, event) -> None:
             self.unit.status = MaintenanceStatus("Starting mysqld")
             logger.info("Data directory is already initialised, skipping configuration")
             self._reconcile_pebble_layer(container)
+            if self.is_new_unit:
+                # when unit is new and has data, it means the app is scaling out
+                # from zero units
+                logger.info("Scaling out from zero units")
+                if self.unit.is_leader():
+                    # create the cluster due it being dissolved on scale-down
+                    self.create_cluster()
+                    self._on_update_status(None)
+                else:
+                    # Non-leader units try to join cluster
+                    self.unit.status = WaitingStatus("Waiting for instance to join the cluster")
+                    self.unit_peer_data.update({
+                        "member-role": "secondary",
+                        "member-state": "waiting",
+                    })
             return
 
         self.unit.status = MaintenanceStatus("Initialising mysqld")
@@ -845,10 +871,12 @@ def _on_update_status(self, _: Optional[UpdateStatusEvent]) -> None:
         if not self.unit.is_leader() and self._is_unit_waiting_to_join_cluster():
             # join cluster test takes precedence over blocked test
             # due to matching criteria
+            logger.info("Attempting to join cluster")
             self.join_unit_to_cluster()
             return
 
         if self._is_cluster_blocked():
+            logger.info("Cluster is blocked. Skipping.")
             return
         del self.restart_peers.data[self.unit]["state"]
 
@@ -926,6 +954,11 @@ def _on_database_storage_detaching(self, _) -> None:
         # Inform other hooks of current status
         self.unit_peer_data["unit-status"] = "removing"
 
+        if self.unit.is_leader():
+            # Update 'units-added-to-cluster' counter in the peer relation databag
+            units = int(self.app_peer_data.get("units-added-to-cluster", 1))
+            self.app_peer_data["units-added-to-cluster"] = str(units - 1)
+
 
 if __name__ == "__main__":
     main(MySQLOperatorCharm)

tests/integration/test_charm.py

@@ -201,6 +201,29 @@ async def test_scale_up_after_scale_down(ops_test: OpsTest) -> None:
         assert len(online_member_addresses) == 3
 
 
+@pytest.mark.group(1)
+@pytest.mark.abort_on_fail
+async def test_scale_up_from_zero(ops_test: OpsTest) -> None:
+    """Ensure scaling down to zero and back up works."""
+    await scale_application(ops_test, APP_NAME, 0)
+
+    await ops_test.model.block_until(
+        lambda: len(ops_test.model.applications[APP_NAME].units) == 0,
+        timeout=TIMEOUT,
+    )
+
+    await scale_application(ops_test, APP_NAME, 3)
+
+    random_unit = ops_test.model.applications[APP_NAME].units[0]
+    cluster_status = await get_cluster_status(random_unit)
+    online_member_addresses = [
+        member["address"]
+        for _, member in cluster_status["defaultreplicaset"]["topology"].items()
+        if member["status"] == "online"
+    ]
+    assert len(online_member_addresses) == 3
+
+
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_password_rotation(ops_test: OpsTest):