DPE-5582 Timeout node count query (#514)

* lib bump

* timeout for node count retrieval

* add method required by lib update

* remove unstable flag

* fix test after change in log level

* allow idle state detection

Author: paulomach

lib/charms/mysql/v0/mysql.py

@@ -134,7 +134,7 @@ def wait_until_mysql_connection(self) -> None:
 # Increment this major API version when introducing breaking changes
 LIBAPI = 0
 
-LIBPATCH = 73
+LIBPATCH = 75
 
 UNIT_TEARDOWN_LOCKNAME = "unit-teardown"
 UNIT_ADD_LOCKNAME = "unit-add"
@@ -1085,16 +1085,25 @@ def configure_mysql_users(self, password_needed: bool = True) -> None:
             )
             raise MySQLConfigureMySQLUsersError(e.message)
 
+    def _plugin_file_exists(self, plugin_file_name: str) -> bool:
+        """Check if the plugin file exists.
+
+        Args:
+            plugin_file_name: Plugin file name, with the extension.
+
+        """
+        path = self.get_variable_value("plugin_dir")
+        return self._file_exists(f"{path}/{plugin_file_name}")
+
     def install_plugins(self, plugins: list[str]) -> None:
         """Install extra plugins."""
         supported_plugins = {
-            "audit_log": "INSTALL PLUGIN audit_log SONAME 'audit_log.so';",
-            "audit_log_filter": "INSTALL PLUGIN audit_log_filter SONAME 'audit_log_filter.so';",
+            "audit_log": ("INSTALL PLUGIN audit_log SONAME", "audit_log.so"),
+            "audit_log_filter": ("INSTALL PLUGIN audit_log_filter SONAME", "audit_log_filter.so"),
         }
 
-        super_read_only = self.get_variable_value("super_read_only").lower() == "on"
-
         try:
+            super_read_only = self.get_variable_value("super_read_only").lower() == "on"
             installed_plugins = self._get_installed_plugins()
             # disable super_read_only to install plugins
             for plugin in plugins:
@@ -1106,7 +1115,16 @@ def install_plugins(self, plugins: list[str]) -> None:
                     logger.warning(f"{plugin=} is not supported")
                     continue
 
-                command = supported_plugins[plugin]
+                command_prefix, plugin_file = (
+                    supported_plugins[plugin][0],
+                    supported_plugins[plugin][1],
+                )
+
+                if not self._plugin_file_exists(plugin_file):
+                    logger.warning(f"{plugin=} file not found. Skip installation")
+                    continue
+
+                command = f"{command_prefix} '{plugin_file}';"
                 if super_read_only:
                     command = (
                         f"SET GLOBAL super_read_only=OFF; {command}"
@@ -1123,6 +1141,10 @@ def install_plugins(self, plugins: list[str]) -> None:
                 f"Failed to install {plugin=}",  # type: ignore
             )
             raise MySQLPluginInstallError
+        except MySQLGetVariableError:
+            # workaround for config changed triggered after failed upgrade
+            # the check fails for charms revisions not using admin address
+            logger.warning("Failed to get super_read_only variable. Skip plugin installation")
 
     def uninstall_plugins(self, plugins: list[str]) -> None:
         """Uninstall plugins."""
@@ -1915,7 +1937,7 @@ def get_cluster_node_count(
         )
 
         try:
-            output = self._run_mysqlsh_script("\n".join(size_commands))
+            output = self._run_mysqlsh_script("\n".join(size_commands), timeout=30)
         except MySQLClientError:
             logger.warning("Failed to get node count")
             return 0
@@ -3147,3 +3169,12 @@ def _run_mysqlcli_script(
             timeout: (optional) time before the query should timeout
         """
         raise NotImplementedError
+
+    @abstractmethod
+    def _file_exists(self, path: str) -> bool:
+        """Check if a file exists.
+
+        Args:
+            path: Path to the file to check
+        """
+        raise NotImplementedError

src/mysql_k8s_helpers.py

@@ -812,3 +812,11 @@ def set_cluster_primary(self, new_primary_address: str) -> None:
     def fetch_error_log(self) -> Optional[str]:
         """Fetch the MySQL error log."""
         return self.read_file_content("/var/log/mysql/error.log")
+
+    def _file_exists(self, path: str) -> bool:
+        """Check if a file exists.
+
+        Args:
+            path: Path to the file to check
+        """
+        return self.container.exists(path)

tests/integration/high_availability/high_availability_helpers.py

@@ -495,10 +495,9 @@ async def ensure_all_units_continuous_writes_incrementing(
     )
 
     async with ops_test.fast_forward(fast_interval="15s"):
-        for attempt in Retrying(stop=stop_after_delay(15 * 60), wait=wait_fixed(10)):
-            with attempt:
-                # ensure that all units are up to date (including the previous primary)
-                for unit in mysql_units:
+        for unit in mysql_units:
+            for attempt in Retrying(stop=stop_after_delay(15 * 60), wait=wait_fixed(10)):
+                with attempt:
                     # ensure the max written value is incrementing (continuous writes is active)
                     max_written_value = await get_max_written_value_in_database(
                         ops_test, unit, credentials

tests/integration/high_availability/test_node_drain.py

@@ -55,7 +55,7 @@ async def test_pod_eviction_and_pvc_deletion(
     delete_pvcs(primary_pod_pvcs)
     delete_pvs(primary_pod_pvs)
 
-    async with ops_test.fast_forward():
+    async with ops_test.fast_forward("90s"):
         logger.info("Waiting for evicted primary pod to be rescheduled")
         await ops_test.model.wait_for_idle(
             apps=[mysql_application_name],

tests/integration/high_availability/test_self_healing.py

@@ -109,7 +109,6 @@ async def test_kill_db_process(
 
 @pytest.mark.group(2)
 @pytest.mark.abort_on_fail
-@pytest.mark.unstable
 async def test_freeze_db_process(
     ops_test: OpsTest, highly_available_cluster, continuous_writes, credentials
 ) -> None:

tests/integration/high_availability/test_upgrade_rollback_incompat.py

@@ -83,7 +83,7 @@ async def test_upgrade_to_failling(ops_test: OpsTest) -> None:
     logger.info("Build charm locally")
 
     sub_regex_failing_rejoin = (
-        's/logger.debug("Recovering unit")'
+        's/logger.info("Recovering unit")'
         '/self.charm._mysql.set_instance_offline_mode(True); raise RetryError("dummy")/'
     )
     src_patch(sub_regex=sub_regex_failing_rejoin, file_name="src/upgrade.py")