DPE-3865 misc test fixes (#480)

* increased timeout on clear

* wait app be active

* mistakenly removed flag

* fixes for rollback from incompatible data_dir

Author: paulomach

src/mysql_vm_helpers.py

@@ -36,6 +36,7 @@
 from constants import (
     CHARMED_MYSQL,
     CHARMED_MYSQL_COMMON_DIRECTORY,
+    CHARMED_MYSQL_DATA_DIRECTORY,
     CHARMED_MYSQL_SNAP_NAME,
     CHARMED_MYSQL_XBCLOUD_LOCATION,
     CHARMED_MYSQL_XBSTREAM_LOCATION,
@@ -217,8 +218,9 @@ def uninstall_mysql() -> None:
                 # uninstalls fail due to SNAP_DATA_DIR fails to umount
                 # try umount it, without check
                 subprocess.run(["umount", CHARMED_MYSQL_COMMON_DIRECTORY])
+                shutil.rmtree(f"{CHARMED_MYSQL_DATA_DIRECTORY}/etc", ignore_errors=True)
                 logger.exception(f"Failed to uninstall MySQL on {attempt=}")
-        raise MySQLUninstallError
+        raise MySQLUninstallError from None
 
     @override
     def get_available_memory(self) -> int:

src/upgrade.py

@@ -7,6 +7,7 @@
 import json
 import logging
 import pathlib
+import subprocess
 from typing import TYPE_CHECKING
 
 from charms.data_platform_libs.v0.upgrade import (
@@ -55,6 +56,12 @@ def get_mysql_dependencies_model() -> MySQLVMDependenciesModel:
     return MySQLVMDependenciesModel(**_deps)
 
 
+def set_cron_daemon(action: str) -> None:
+    """Start/stop the cron daemon."""
+    logger.debug(f"{action}ing cron daemon")
+    subprocess.run(["systemctl", action, "cron"], check=True)
+
+
 class MySQLVMUpgrade(DataUpgrade):
     """MySQL upgrade class."""
 
@@ -189,6 +196,8 @@ def _on_upgrade_granted(self, event: UpgradeGrantedEvent) -> None:  # noqa: C901
                 memory_limit=self.charm.config.profile_limit_memory,
             )
             self.charm.unit.status = MaintenanceStatus("starting services...")
+            # stop cron daemon to be able to query `error.log`
+            set_cron_daemon("stop")
             self.charm._mysql.start_mysqld()
             self.charm._mysql.setup_logrotate_and_cron()
         except VersionError:
@@ -215,6 +224,8 @@ def _on_upgrade_granted(self, event: UpgradeGrantedEvent) -> None:  # noqa: C901
             logger.exception("Failed to stop MySQL server")
             self.set_unit_failed()
             return
+        finally:
+            set_cron_daemon("start")
 
         try:
             self.charm.unit.set_workload_version(self.charm._mysql.get_mysql_version() or "unset")

tests/integration/helpers.py

@@ -266,7 +266,7 @@ def is_relation_broken(ops_test: OpsTest, endpoint_one: str, endpoint_two: str)
     return False
 
 
-@retry(stop=stop_after_attempt(8), wait=wait_fixed(15), reraise=True)
+@retry(stop=stop_after_attempt(30), wait=wait_fixed(5), reraise=True)
 def is_connection_possible(
     credentials: Dict, *, retry_if_not_possible=False, **extra_opts
 ) -> bool:

tests/integration/high_availability/conftest.py

@@ -21,7 +21,7 @@ async def continuous_writes(ops_test: OpsTest):
     application_unit = ops_test.model.applications[application_name].units[0]
 
     logger.info("Clearing continuous writes")
-    await juju_.run_action(application_unit, "clear-continuous-writes")
+    await juju_.run_action(application_unit, "clear-continuous-writes", **{"timeout": 120})
 
     logger.info("Starting continuous writes")
     await juju_.run_action(application_unit, "start-continuous-writes")

tests/integration/high_availability/test_async_replication.py

@@ -198,10 +198,8 @@ async def test_deploy_router_and_app(first_model: Model) -> None:
     logger.info("Relate router and db")
     await first_model.integrate(MYSQL_ROUTER_APP_NAME, MYSQL_APP1)
 
-    await first_model.wait_for_idle(
-        apps=[MYSQL_ROUTER_APP_NAME, APPLICATION_APP_NAME],
-        timeout=10 * MINUTE,
-        raise_on_error=False,
+    await first_model.block_until(
+        lambda: first_model.applications[APPLICATION_APP_NAME].units[0].workload_status == "active"
     )
 
 

tests/integration/high_availability/test_upgrade_rollback_incompat.py

@@ -140,7 +140,6 @@ async def test_upgrade_to_failling(
 # (details: https://github.com/canonical/mysql-operator/pull/472#discussion_r1659300069)
 @markers.amd64_only
 @pytest.mark.abort_on_fail
-@pytest.mark.unstable
 async def test_rollback(ops_test, continuous_writes) -> None:
     application = ops_test.model.applications[MYSQL_APP_NAME]
 
@@ -152,7 +151,7 @@ async def test_rollback(ops_test, continuous_writes) -> None:
     new_revisions["x86_64"] = "69"
     with snap_revisions.open("w") as file:
         json.dump(new_revisions, file)
-    charm = await charm_local_build(ops_test)
+    charm = await charm_local_build(ops_test, refresh=True)
 
     logger.info("Get leader unit")
     leader_unit = await get_leader_unit(ops_test, MYSQL_APP_NAME)

tests/unit/test_upgrade.py

@@ -131,6 +131,7 @@ def test_pre_upgrade_prepare(
         mock_get_primary_label.assert_called_once()
         assert mock_set_dynamic_variable.call_count == 2
 
+    @patch("upgrade.set_cron_daemon")
     @patch("mysql_vm_helpers.MySQL.write_mysqld_config")
     @patch("upgrade.MySQLVMUpgrade._check_server_unsupported_downgrade")
     @patch("upgrade.MySQLVMUpgrade._reset_on_unsupported_downgrade")
@@ -160,6 +161,7 @@ def test_upgrade_granted(
         mock_reset_on_unsupported_downgrade,
         mock_check_server_unsupported_downgrade,
         mock_write_mysqld_config,
+        mock_set_cron_daemon,
     ):
         """Test upgrade-granted hook."""
         self.charm.on.config_changed.emit()