[MISC] Don't restart during initial sync (#1000)

* Don't restart during initial sync

* Catch relations map exceptions

Author: dragomirp

src/charm.py

@@ -2157,35 +2157,43 @@ def relations_user_databases_map(self) -> dict:
         if not self.is_cluster_initialised or not self._patroni.member_started:
             return {USER: "all", REPLICATION_USER: "all", REWIND_USER: "all"}
         user_database_map = {}
-        for user in sorted(
-            self.postgresql.list_users_from_relation(current_host=self.is_connectivity_enabled)
-        ):
-            user_database_map[user] = ",".join(
-                sorted(
-                    self.postgresql.list_accessible_databases_for_user(
-                        user, current_host=self.is_connectivity_enabled
+        try:
+            for user in sorted(
+                self.postgresql.list_users_from_relation(current_host=self.is_connectivity_enabled)
+            ):
+                user_database_map[user] = ",".join(
+                    sorted(
+                        self.postgresql.list_accessible_databases_for_user(
+                            user, current_host=self.is_connectivity_enabled
+                        )
                     )
                 )
-            )
-            # Add "landscape" superuser by default to the list when the "db-admin" relation is present
-            # or when the "database" relation has "extra-user-roles" set to "SUPERUSER" (which may mean
-            # that PgBouncer is related to the database and there is the possibility that Landscape
-            # is related to it).
-            if any(
-                True
-                for relation in self.client_relations
-                if relation.name == "db-admin"  # Possibly Landscape relation.
-                or (
-                    relation.name == "database"
-                    and relation.data[relation.app].get("extra-user-roles") == "SUPERUSER"
-                )  # PgBouncer (which may be related to Landscape).
-            ):
-                user_database_map["landscape"] = "all"
-        if self.postgresql.list_access_groups(current_host=self.is_connectivity_enabled) != set(
-            ACCESS_GROUPS
-        ):
-            user_database_map.update({USER: "all", REPLICATION_USER: "all", REWIND_USER: "all"})
-        return user_database_map
+                # Add "landscape" superuser by default to the list when the "db-admin" relation is present
+                # or when the "database" relation has "extra-user-roles" set to "SUPERUSER" (which may mean
+                # that PgBouncer is related to the database and there is the possibility that Landscape
+                # is related to it).
+                if any(
+                    True
+                    for relation in self.client_relations
+                    if relation.name == "db-admin"  # Possibly Landscape relation.
+                    or (
+                        relation.name == "database"
+                        and relation.data[relation.app].get("extra-user-roles") == "SUPERUSER"
+                    )  # PgBouncer (which may be related to Landscape).
+                ):
+                    user_database_map["landscape"] = "all"
+            if self.postgresql.list_access_groups(
+                current_host=self.is_connectivity_enabled
+            ) != set(ACCESS_GROUPS):
+                user_database_map.update({
+                    USER: "all",
+                    REPLICATION_USER: "all",
+                    REWIND_USER: "all",
+                })
+            return user_database_map
+        except PostgreSQLListUsersError:
+            logger.debug("relations_user_databases_map: Unable to get users")
+            return {USER: "all", REPLICATION_USER: "all", REWIND_USER: "all"}
 
     def override_patroni_restart_condition(
         self, new_condition: str, repeat_cause: str | None

src/cluster.py

@@ -53,7 +53,8 @@
 
 PG_BASE_CONF_PATH = f"{POSTGRESQL_CONF_PATH}/postgresql.conf"
 
-RUNNING_STATES = ["running", "streaming"]
+STARTED_STATES = ["running", "streaming"]
+RUNNING_STATES = [*STARTED_STATES, "starting"]
 
 PATRONI_TIMEOUT = 10
 
@@ -336,7 +337,7 @@ def get_standby_leader(
                 )
                 for member in cluster_status.json()["members"]:
                     if member["role"] == "standby_leader":
-                        if check_whether_is_running and member["state"] not in RUNNING_STATES:
+                        if check_whether_is_running and member["state"] not in STARTED_STATES:
                             logger.warning(f"standby leader {member['name']} is not running")
                             continue
                         standby_leader = member["name"]
@@ -416,7 +417,7 @@ def are_all_members_ready(self) -> bool:
         # a standby leader, because sometimes there may exist (for some period of time)
         # only replicas after a failed switchover.
         return all(
-            member["state"] in RUNNING_STATES for member in cluster_status.json()["members"]
+            member["state"] in STARTED_STATES for member in cluster_status.json()["members"]
         ) and any(
             member["role"] in ["leader", "standby_leader"]
             for member in cluster_status.json()["members"]
@@ -496,6 +497,8 @@ def member_started(self) -> bool:
             True if services is ready False otherwise. Retries over a period of 60 seconds times to
             allow server time to start up.
         """
+        if not self.is_patroni_running():
+            return False
         try:
             response = self.get_patroni_health()
         except RetryError:
@@ -517,7 +520,7 @@ def member_inactive(self) -> bool:
             return True
 
         return response["state"] not in [
-            *RUNNING_STATES,
+            *STARTED_STATES,
             "creating replica",
             "starting",
             "restarting",
@@ -965,6 +968,16 @@ def reload_patroni_configuration(self):
             timeout=PATRONI_TIMEOUT,
         )
 
+    def is_patroni_running(self) -> bool:
+        """Check if the Patroni service is running."""
+        try:
+            cache = snap.SnapCache()
+            selected_snap = cache["charmed-postgresql"]
+            return selected_snap.services["patroni"]["active"]
+        except snap.SnapError as e:
+            logger.debug(f"Failed to check Patroni service: {e}")
+            return False
+
     def restart_patroni(self) -> bool:
         """Restart Patroni.
 

tests/unit/test_cluster.py

@@ -571,6 +571,7 @@ def test_member_started_true(peers_ips, patroni):
         patch("cluster.requests.get") as _get,
         patch("cluster.stop_after_delay", return_value=stop_after_delay(0)),
         patch("cluster.wait_fixed", return_value=wait_fixed(0)),
+        patch("charm.Patroni.is_patroni_running", return_value=True),
     ):
         _get.return_value.json.return_value = {"state": "running"}
 
@@ -586,6 +587,7 @@ def test_member_started_false(peers_ips, patroni):
         patch("cluster.requests.get") as _get,
         patch("cluster.stop_after_delay", return_value=stop_after_delay(0)),
         patch("cluster.wait_fixed", return_value=wait_fixed(0)),
+        patch("charm.Patroni.is_patroni_running", return_value=True),
     ):
         _get.return_value.json.return_value = {"state": "stopped"}
 
@@ -601,6 +603,7 @@ def test_member_started_error(peers_ips, patroni):
         patch("cluster.requests.get") as _get,
         patch("cluster.stop_after_delay", return_value=stop_after_delay(0)),
         patch("cluster.wait_fixed", return_value=wait_fixed(0)),
+        patch("charm.Patroni.is_patroni_running", return_value=True),
     ):
         _get.side_effect = Exception