DPE-7726: Avoid pending_restart=True flag flickering

The current Patroni 3.2.2 has wired/flickering  behaviour:
it temporary flag pending_restart=True on many changes to REST API,
which is gone within a second but long enough to be cougth by charm.
Sleepping a bit is a necessary evil, until Patroni 3.3.0 upgrade.

The previous code sleept for 15 seconds waiting for pg_settings update.

Also, the unnecessary restarts could be triggered by missmatch of
Patroni config file and in-memory changes coming from REST API,
e.g. the slots were undefined in yaml file but set as an empty JSON {} => None.
Updating the default template to match the default API PATCHes and avoid restarts.

Author: taurus-forever

src/charm.py

@@ -2365,7 +2365,7 @@ def update_config(
             parameters=pg_parameters,
             no_peers=no_peers,
             user_databases_map=self.relations_user_databases_map,
-            slots=replication_slots or None,
+            slots=replication_slots,
         )
         if no_peers:
             return True

src/cluster.py

@@ -16,6 +16,7 @@
 from contextlib import suppress
 from pathlib import Path
 from ssl import CERT_NONE, create_default_context
+from time import sleep
 from typing import TYPE_CHECKING, Any, TypedDict
 
 import psutil
@@ -466,18 +467,32 @@ def get_patroni_health(self) -> dict[str, str]:
 
     def is_restart_pending(self) -> bool:
         """Returns whether the Patroni/PostgreSQL restart pending."""
-        patroni_status = requests.get(
+        pending_restart = self._get_patroni_restart_pending()
+        if pending_restart:
+            # The current Patroni 3.2.2 has wired behaviour: it temporary flag pending_restart=True
+            # on any changes to REST API, which is gone within a second but long enough to be
+            # cougth by charm. Sleep 2 seconds as a protection here until Patroni 3.3.0 upgrade.
+            # Repeat the request to make sure pending_restart flag is still here
+            logger.debug("Enduring restart is pending (to avoid unnecessary rolling restarts)")
+            sleep(2)
+            pending_restart = self._get_patroni_restart_pending()
+
+        return pending_restart
+
+    def _get_patroni_restart_pending(self) -> bool:
+        """Returns whether the Patroni flag pending_restart on REST API."""
+        r = requests.get(
             f"{self._patroni_url}/patroni",
             verify=self.verify,
             timeout=API_REQUEST_TIMEOUT,
             auth=self._patroni_auth,
         )
-        try:
-            pending_restart = patroni_status.json()["pending_restart"]
-        except KeyError:
-            pending_restart = False
-            pass
-        logger.debug(f"Patroni API is_restart_pending: {pending_restart}")
+        pending_restart = r.json().get("pending_restart", False)
+        logger.debug(
+            f"API _get_patroni_restart_pending ({pending_restart}): %s (%s)",
+            r,
+            r.elapsed.total_seconds(),
+        )
 
         return pending_restart
 
@@ -1083,7 +1098,7 @@ def ensure_slots_controller_by_patroni(self, slots: dict[str, str]) -> None:
                         f"Failed to get current Patroni config: {current_config.status_code} {current_config.text}"
                     )
         slots_patch: dict[str, dict[str, str] | None] = dict.fromkeys(
-            current_config.json().get("slots", ())
+            current_config.json().get("slots", ()) or {}
         )
         for slot, database in slots.items():
             slots_patch[slot] = {

templates/patroni.yml.j2

@@ -101,15 +101,13 @@ bootstrap:
         set_user.block_log_statement: 'on'
         set_user.exit_on_error: 'on'
         set_user.superuser_allowlist: '+charmed_dba'
-    {%- if slots %}
     slots:
     {%- for slot, database in slots.items() %}
       {{slot}}:
         database: {{database}}
         plugin: pgoutput
         type: logical
     {%- endfor -%}
-    {% endif %}
 
   {%- if restoring_backup %}
   method: pgbackrest
@@ -202,7 +200,7 @@ postgresql:
     {%- endfor %}
     {%- for peer_ip in peers_ips %}
     - {{ 'hostssl' if enable_tls else 'host' }}     replication    replication    {{ peer_ip }}/0    md5
-    {% endfor %}
+    {%- endfor %}
   pg_ident:
   - operator _daemon_ backup
   authentication:

tests/unit/test_charm.py

@@ -1162,7 +1162,7 @@ class _MockSnap:
             parameters={"test": "test"},
             no_peers=False,
             user_databases_map={"operator": "all", "replication": "all", "rewind": "all"},
-            slots=None,
+            slots={},
         )
         _handle_postgresql_restart_need.assert_called_once_with()
         _restart_ldap_sync_service.assert_called_once()
@@ -1193,7 +1193,7 @@ class _MockSnap:
             parameters={"test": "test"},
             no_peers=False,
             user_databases_map={"operator": "all", "replication": "all", "rewind": "all"},
-            slots=None,
+            slots={},
         )
         _handle_postgresql_restart_need.assert_called_once()
         _restart_ldap_sync_service.assert_called_once()