[DPE-6488] Port over mysql charmlib changes from vm operator related to unit initialized (#593)

* Port over mysql charmlib changes from vm operator

* Remove redundant check when executing manual rejoin

* Update to latest the mysql charm lib

* Attempt to make scale detection faster and easier in crash during setup test

Author: shayancanonical

lib/charms/mysql/v0/async_replication.py

@@ -730,7 +730,7 @@ def _on_consumer_relation_created(self, event):
         """Handle the async_replica relation being created on the leader unit."""
         if not self._charm.unit.is_leader():
             return
-        if not self._charm.unit_initialized and not self.returning_cluster:
+        if not self._charm.unit_initialized() and not self.returning_cluster:
             # avoid running too early for non returning clusters
             self._charm.unit.status = BlockedStatus(
                 "Wait until unit is initialized before running create-replication on offer side"

lib/charms/mysql/v0/mysql.py

@@ -133,7 +133,7 @@ def wait_until_mysql_connection(self) -> None:
 # Increment this major API version when introducing breaking changes
 LIBAPI = 0
 
-LIBPATCH = 84
+LIBPATCH = 85
 
 UNIT_TEARDOWN_LOCKNAME = "unit-teardown"
 UNIT_ADD_LOCKNAME = "unit-add"
@@ -418,6 +418,10 @@ class MySQLPluginInstallError(Error):
     """Exception raised when there is an issue installing a MySQL plugin."""
 
 
+class MySQLClusterMetadataExistsError(Error):
+    """Exception raised when there is an issue checking if cluster metadata exists."""
+
+
 @dataclasses.dataclass
 class RouterUser:
     """MySQL Router user."""
@@ -618,8 +622,15 @@ def cluster_initialized(self) -> bool:
             return False
 
         for unit in self.app_units:
-            if self._mysql.cluster_metadata_exists(self.get_unit_address(unit)):
-                return True
+            try:
+                if unit != self.unit and self._mysql.cluster_metadata_exists(
+                    self.get_unit_address(unit)
+                ):
+                    return True
+                elif self._mysql.cluster_metadata_exists():
+                    return True
+            except MySQLClusterMetadataExistsError:
+                pass
 
         return False
 
@@ -660,11 +671,6 @@ def unit_configured(self) -> bool:
             self.get_unit_address(self.unit), self.unit_label
         )
 
-    @property
-    def unit_initialized(self) -> bool:
-        """Check if the unit is added to the cluster."""
-        return self._mysql.cluster_metadata_exists(self.get_unit_address(self.unit))
-
     @property
     def app_peer_data(self) -> Union[ops.RelationDataContent, dict]:
         """Application peer relation data object."""
@@ -743,6 +749,15 @@ def removing_unit(self) -> bool:
         """Check if the unit is being removed."""
         return self.unit_peer_data.get("unit-status") == "removing"
 
+    def unit_initialized(self, raise_exceptions: bool = False) -> bool:
+        """Check if the unit is added to the cluster."""
+        try:
+            return self._mysql.cluster_metadata_exists()
+        except MySQLClusterMetadataExistsError:
+            if raise_exceptions:
+                raise
+            return False
+
     def peer_relation_data(self, scope: Scopes) -> DataPeerData:
         """Returns the peer relation data per scope."""
         if scope == APP_SCOPE:
@@ -1671,35 +1686,62 @@ def is_cluster_in_cluster_set(self, cluster_name: str) -> Optional[bool]:
 
         return cluster_name in cs_status["clusters"]
 
-    def cluster_metadata_exists(self, from_instance: str) -> bool:
-        """Check if this cluster metadata exists on database."""
-        check_cluster_metadata_commands = (
-            "result = session.run_sql(\"SHOW DATABASES LIKE 'mysql_innodb_cluster_metadata'\")",
-            "content = result.fetch_all()",
-            "if content:",
-            (
-                '  result = session.run_sql("SELECT cluster_name FROM mysql_innodb_cluster_metadata'
-                f".clusters where cluster_name = '{self.cluster_name}';\")"
-            ),
-            "  print(bool(result.fetch_one()))",
-            "else:",
-            "  print(False)",
+    def cluster_metadata_exists(self, from_instance: Optional[str] = None) -> bool:
+        """Check if this cluster metadata exists on database.
+
+        Use mysqlsh when querying clusters from remote instances. However, use
+        mysqlcli when querying locally since this method can be called before
+        the cluster is initialized (before serverconfig and root users are set up
+        correctly)
+        """
+        get_clusters_query = (
+            "SELECT cluster_name "
+            "FROM mysql_innodb_cluster_metadata.clusters "
+            "WHERE EXISTS ("
+            "SELECT * "
+            "FROM information_schema.schemata "
+            "WHERE schema_name = 'mysql_innodb_cluster_metadata'"
+            ")"
         )
 
+        if from_instance:
+            check_cluster_metadata_commands = (
+                f'cursor = session.run_sql("{get_clusters_query}")',
+                "print(cursor.fetch_all())",
+            )
+
+            try:
+                output = self._run_mysqlsh_script(
+                    "\n".join(check_cluster_metadata_commands),
+                    user=self.server_config_user,
+                    password=self.server_config_password,
+                    host=self.instance_def(self.server_config_user, from_instance),
+                    timeout=60,
+                    exception_as_warning=True,
+                )
+            except MySQLClientError:
+                logger.warning(f"Failed to check if cluster metadata exists {from_instance=}")
+                raise MySQLClusterMetadataExistsError(
+                    f"Failed to check if cluster metadata exists {from_instance=}"
+                )
+
+            return self.cluster_name in output
+
         try:
-            output = self._run_mysqlsh_script(
-                "\n".join(check_cluster_metadata_commands),
-                user=self.server_config_user,
-                password=self.server_config_password,
-                host=self.instance_def(self.server_config_user, from_instance),
+            output = self._run_mysqlcli_script(
+                (get_clusters_query,),
+                user=ROOT_USERNAME,
+                password=self.root_password,
                 timeout=60,
                 exception_as_warning=True,
+                log_errors=False,
             )
         except MySQLClientError:
-            logger.warning(f"Failed to check if cluster metadata exists {from_instance=}")
-            return False
+            logger.warning("Failed to check if local cluster metadata exists")
+            raise MySQLClusterMetadataExistsError("Failed to check if cluster metadata exists")
 
-        return output.strip() == "True"
+        cluster_names = [entry[0].strip() for entry in output]
+        return self.cluster_name in cluster_names
 
     def rejoin_cluster(self, cluster_name) -> None:
         """Try to rejoin a cluster to the cluster set."""
@@ -1939,8 +1981,11 @@ def rescan_cluster(
 
     def is_instance_in_cluster(self, unit_label: str) -> bool:
         """Confirm if instance is in the cluster."""
-        if not self.cluster_metadata_exists(self.instance_address):
-            # early return if instance has no cluster metadata
+        try:
+            if not self.cluster_metadata_exists(self.instance_address):
+                # early return if instance has no cluster metadata
+                return False
+        except MySQLClusterMetadataExistsError:
             return False
 
         commands = (
@@ -3244,21 +3289,26 @@ def kill_client_sessions(self) -> None:
             logger.error("Failed to kill external sessions")
             raise MySQLKillSessionError
 
-    def check_mysqlsh_connection(self) -> bool:
-        """Checks if it is possible to connect to the server with mysqlsh."""
-        connect_commands = 'session.run_sql("SELECT 1")'
+    def check_mysqlcli_connection(self) -> bool:
+        """Checks if it is possible to connect to the server with mysqlcli."""
+        connect_commands = ("SELECT 1",)
 
         try:
-            self._run_mysqlsh_script(
+            self._run_mysqlcli_script(
                 connect_commands,
                 user=self.server_config_user,
                 password=self.server_config_password,
-                host=self.instance_def(self.server_config_user),
             )
             return True
         except MySQLClientError:
-            logger.error("Failed to connect to MySQL with mysqlsh")
-            return False
+            logger.warning("Failed to connect to MySQL with mysqlcli with server config user")
+
+            try:
+                self._run_mysqlcli_script(connect_commands)
+                return True
+            except MySQLClientError:
+                logger.error("Failed to connect to MySQL with mysqlcli with default root user")
+                return False
 
     def get_pid_of_port_3306(self) -> Optional[str]:
         """Retrieves the PID of the process that is bound to port 3306."""
@@ -3427,6 +3477,7 @@ def _run_mysqlcli_script(
         password: Optional[str] = None,
         timeout: Optional[int] = None,
         exception_as_warning: bool = False,
+        log_errors: bool = False,
     ) -> list:
         """Execute a MySQL CLI script.
 
@@ -3442,6 +3493,7 @@ def _run_mysqlcli_script(
             password: (optional) password to invoke the mysql cli script with
             timeout: (optional) time before the query should timeout
             exception_as_warning: (optional) whether the exception should be treated as warning
+            log_errors: (optional) whether errors in the output should be logged
         """
         raise NotImplementedError
 

lib/charms/mysql/v0/tls.py

@@ -91,7 +91,7 @@ def _on_set_tls_private_key(self, event: ActionEvent) -> None:
 
     def _on_tls_relation_joined(self, event) -> None:
         """Request certificate when TLS relation joined."""
-        if not self.charm.unit_initialized:
+        if not self.charm.unit_initialized():
             event.defer()
             return
         self._request_certificate(None)

src/charm.py

@@ -302,15 +302,14 @@ def text_logs(self) -> list:
 
         return text_logs
 
-    @property
-    def unit_initialized(self) -> bool:
+    def unit_initialized(self, raise_exceptions: bool = False) -> bool:
         """Return whether a unit is started.
 
         Override parent class method to include container accessibility check.
         """
         container = self.unit.get_container(CONTAINER_NAME)
         if container.can_connect():
-            return super().unit_initialized
+            return super().unit_initialized()
         else:
             return False
 
@@ -367,7 +366,7 @@ def _is_unit_waiting_to_join_cluster(self) -> bool:
             and self.unit_peer_data.get("member-state") == "waiting"
             and self.unit_configured
             and (
-                not self.unit_initialized
+                not self.unit_initialized()
                 or not self._mysql.is_instance_in_cluster(self.unit_label)
             )
             and self.cluster_initialized
@@ -505,7 +504,7 @@ def _restart(self, _: EventBase) -> None:
         if not container.can_connect():
             return
 
-        if not self.unit_initialized:
+        if not self.unit_initialized():
             logger.debug("Restarting standalone mysqld")
             container.restart(MYSQLD_SERVICE)
             return
@@ -1013,7 +1012,7 @@ def _on_peer_relation_changed(self, event: RelationChangedEvent) -> None:
     def _on_database_storage_detaching(self, _) -> None:
         """Handle the database storage detaching event."""
         # Only executes if the unit was initialised
-        if not self.unit_initialized:
+        if not self.unit_initialized():
             return
 
         # No need to remove the instance from the cluster if it is not a member of the cluster

src/log_rotate_manager.py

@@ -38,7 +38,7 @@ def start_log_rotate_manager(self):
             not isinstance(self.charm.unit.status, ActiveStatus)
             or self.charm.peers is None
             or not container.can_connect()
-            or not self.charm.unit_initialized
+            or not self.charm.unit_initialized()
         ):
             return
 

src/mysql_k8s_helpers.py

@@ -281,8 +281,8 @@ def wait_until_mysql_connection(self, check_port: bool = True) -> None:
             raise MySQLServiceNotRunningError
 
         try:
-            if check_port and not self.check_mysqlsh_connection():
-                raise MySQLServiceNotRunningError("Connection with mysqlsh not possible")
+            if check_port and not self.check_mysqlcli_connection():
+                raise MySQLServiceNotRunningError("Connection with mysqlcli not possible")
         except MySQLClientError:
             raise MySQLServiceNotRunningError
 
@@ -736,6 +736,7 @@ def _run_mysqlcli_script(
         password: Optional[str] = None,
         timeout: Optional[int] = None,
         exception_as_warning: bool = False,
+        **_,
     ) -> list:
         """Execute a MySQL CLI script.
 

src/relations/mysql.py

@@ -140,7 +140,7 @@ def _update_status(self, _) -> None:
         if not container.can_connect():
             return
 
-        if not self.charm.unit_initialized:
+        if not self.charm.unit_initialized():
             # Skip update status for uninitialized unit
             return
 
@@ -168,7 +168,7 @@ def _on_peer_relation_changed(self, event: RelationChangedEvent) -> None:
         if not self.model.get_relation(LEGACY_MYSQL):
             return
 
-        if not (self.charm._is_peer_data_set and self.charm.unit_initialized):
+        if not (self.charm._is_peer_data_set and self.charm.unit_initialized()):
             # Avoid running too early
             logger.info("Unit not ready to set `mysql` relation data. Deferring")
             event.defer()
@@ -211,7 +211,7 @@ def _on_mysql_relation_created(self, event: RelationCreatedEvent) -> None:  # no
         # and for the member to be initialized and online
         if (
             not self.charm._is_peer_data_set
-            or not self.charm.unit_initialized
+            or not self.charm.unit_initialized()
             or self.charm.unit_peer_data.get("member-state") != "online"
         ):
             logger.info("Unit not ready to execute `mysql` relation created. Deferring")

src/relations/mysql_provider.py

@@ -217,7 +217,7 @@ def _configure_endpoints(self, _) -> None:
             return
 
         relations = self.charm.model.relations.get(DB_RELATION_NAME, [])
-        if not relations or not self.charm.unit_initialized:
+        if not relations or not self.charm.unit_initialized():
             return
 
         relation_data = self.database.fetch_relation_data()
@@ -248,7 +248,7 @@ def _on_update_status(self, _) -> None:
         if (
             not container.can_connect()
             or not self.charm.cluster_initialized
-            or not self.charm.unit_initialized
+            or not self.charm.unit_initialized()
         ):
             return
 

src/relations/mysql_root.py

@@ -159,7 +159,7 @@ def _on_mysql_root_relation_created(self, event: RelationCreatedEvent) -> None:
 
         # Wait until on-config-changed event is executed
         # (wait for root password to have been set) or wait until the unit is initialized
-        if not (self.charm._is_peer_data_set and self.charm.unit_initialized):
+        if not (self.charm._is_peer_data_set and self.charm.unit_initialized()):
             event.defer()
             return
 

src/rotate_mysql_logs.py

@@ -46,7 +46,7 @@ def _rotate_mysql_logs(self, _) -> None:
         if (
             self.charm.peers is None
             or not self.charm._mysql.is_mysqld_running()
-            or not self.charm.unit_initialized
+            or not self.charm.unit_initialized()
             or not self.charm.upgrade.idle
         ):
             # skip when not initialized, during an upgrade, or when mysqld is not running