fix(backups): always reset the connectivity flag after a replica backup

A replica backup disables external connectivity by writing the unit peer
databag "connectivity" flag to "off". The flag was only reset to "on"
when `not self.charm.is_primary` still held after the long-running backup
run, so a replica→primary failover mid-backup — or any exception or
interruption in the backup run — left the flag stuck "off". Because the
flag lives in the unit peer databag it survives restarts and upgrades,
leaving pg_hba rejecting peer/replica connections indefinitely (the
cluster can no longer talk to itself).

Latch the disabled-connectivity decision before the backup run and reset
it in a finally block, so the reset runs regardless of role changes or
exceptions. Also recover deployments already stuck with a stale "off" by
resetting it in _on_start and _on_config_changed, guarded by the
cluster-wide is_creating_backup signal so a genuinely in-progress backup
is not un-hidden mid-run.

Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>

Author: marceloneppel

src/backups.py

@@ -985,22 +985,28 @@ def _on_create_backup_action(self, event) -> None:
             event.fail(error_message)
             return
 
+        disabled_connectivity = False
         if not self.charm.is_primary:
             # Create a rule to mark the cluster as in a creating backup state and update
             # the Patroni configuration.
             self._change_connectivity_to_database(connectivity=False)
+            disabled_connectivity = True
 
         self.charm.unit.status = MaintenanceStatus("creating backup")
         # Set flag due to missing in progress backups on JSON output
         # (reference: https://github.com/pgbackrest/pgbackrest/issues/2007)
         self.charm.update_config(is_creating_backup=True)
 
-        self._run_backup(event, s3_parameters, datetime_backup_requested, backup_type)
-
-        if not self.charm.is_primary:
-            # Remove the rule that marks the cluster as in a creating backup state
-            # and update the Patroni configuration.
-            self._change_connectivity_to_database(connectivity=True)
+        # Latch the disabled-connectivity decision before the long-running backup so the
+        # reset runs even if this unit is promoted or the action is interrupted — a stale
+        # "off" flag leaves pg_hba rejecting peer/replica connections across restarts.
+        try:
+            self._run_backup(event, s3_parameters, datetime_backup_requested, backup_type)
+        finally:
+            if disabled_connectivity:
+                # Remove the rule that marks the cluster as in a creating backup state
+                # and update the Patroni configuration.
+                self._change_connectivity_to_database(connectivity=True)
 
         self.charm.update_config(is_creating_backup=False)
         self.charm.unit.status = ActiveStatus()

src/charm.py

@@ -1008,6 +1008,20 @@ def is_connectivity_enabled(self) -> bool:
         """Return whether this unit can be connected externally."""
         return self.unit_peer_data.get("connectivity", "on") == "on"
 
+    def reset_stale_connectivity_flag(self) -> None:
+        """Reset a stale "connectivity: off" left by an interrupted backup.
+
+        The flag is written to the unit peer databag during replica backups and survives
+        restarts/upgrades; without this recovery a unit stuck "off" stays externally
+        disconnected (pg_hba rejects peer/replica connections). Skipped while any backup
+        is in progress cluster-wide to avoid un-hiding a unit mid-backup.
+        """
+        if self._patroni.is_creating_backup:
+            return
+        if self.unit_peer_data.get("connectivity") == "off":
+            logger.info("Resetting stale connectivity flag left by an interrupted backup")
+            self.unit_peer_data["connectivity"] = "on"
+
     @property
     def is_ldap_charm_related(self) -> bool:
         """Return whether this unit has an LDAP charm related."""
@@ -1233,6 +1247,9 @@ def _on_config_changed(self, event) -> None:
             logger.debug("Defer on_config_changed: upgrade in progress")
             event.defer()
             return
+        # Recover from a stale "connectivity: off" left by an interrupted backup before
+        # re-rendering the Patroni configuration below.
+        self.reset_stale_connectivity_flag()
         try:
             self._validate_config_options()
             # update config on every run
@@ -1363,6 +1380,10 @@ def _on_start(self, event: StartEvent) -> None:
         if not self._can_start(event):
             return
 
+        # Recover from a stale "connectivity: off" left by an interrupted backup before
+        # the cluster is rendered/started.
+        self.reset_stale_connectivity_flag()
+
         try:
             postgres_password = self._get_password()
         except ModelError:

tests/unit/test_backups.py

@@ -640,6 +640,90 @@ def test_change_connectivity_to_database(harness):
         _update_config.assert_called_once()
 
 
+def test_create_backup_resets_connectivity_when_run_backup_raises(harness):
+    # Regression: if the backup run is interrupted (raises) on a replica, the connectivity
+    # flag must still be reset to "on" via the finally block — otherwise pg_hba keeps
+    # rejecting peer/replica connections across restarts/upgrades.
+    with (
+        patch("charm.PostgresqlOperatorCharm.update_config"),
+        patch(
+            "charm.PostgreSQLBackups._change_connectivity_to_database"
+        ) as _change_connectivity_to_database,
+        patch(
+            "charm.PostgreSQLBackups._run_backup", side_effect=RuntimeError("boom")
+        ) as _run_backup,
+        patch("charm.PostgresqlOperatorCharm.is_primary", new_callable=PropertyMock) as _is_primary,
+        patch("charm.PostgreSQLBackups._upload_content_to_s3", return_value=True),
+        patch("backups.datetime"),
+        patch("ops.JujuVersion.from_environ"),
+        patch("charm.PostgreSQLBackups._retrieve_s3_parameters") as _retrieve_s3_parameters,
+        patch("charm.PostgreSQLBackups._can_unit_perform_backup", return_value=(True, None)),
+    ):
+        # This unit is a replica, so connectivity is disabled before the backup runs.
+        _is_primary.return_value = False
+        _retrieve_s3_parameters.return_value = (
+            {
+                "bucket": "test-bucket",
+                "access-key": "test-access-key",
+                "secret-key": "test-secret-key",
+                "endpoint": "test-endpoint",
+                "path": "test-path",
+                "region": "test-region",
+            },
+            [],
+        )
+        mock_event = MagicMock()
+        mock_event.params = {"type": "full"}
+
+        # The backup run raises; the reset must still run.
+        with pytest.raises(RuntimeError):
+            harness.charm.backup._on_create_backup_action(mock_event)
+
+        # Connectivity disabled then re-enabled despite the exception.
+        _change_connectivity_to_database.assert_has_calls(
+            [call(connectivity=False), call(connectivity=True)]
+        )
+        _run_backup.assert_called_once()
+
+
+def test_reset_stale_connectivity_flag(harness):
+    with patch("charm.PostgresqlOperatorCharm._patroni", new_callable=PropertyMock) as _patroni:
+        peer_rel_id = harness.model.get_relation(PEER).id
+
+        # Stale "off" with no backup in progress: reset to "on".
+        _patroni.return_value.is_creating_backup = False
+        with harness.hooks_disabled():
+            harness.update_relation_data(
+                peer_rel_id, harness.charm.unit.name, {"connectivity": "off"}
+            )
+        harness.charm.reset_stale_connectivity_flag()
+        assert (
+            harness.get_relation_data(peer_rel_id, harness.charm.unit)["connectivity"] == "on"
+        )
+
+        # While a backup is in progress cluster-wide: do not reset (stay "off").
+        with harness.hooks_disabled():
+            harness.update_relation_data(
+                peer_rel_id, harness.charm.unit.name, {"connectivity": "off"}
+            )
+        _patroni.return_value.is_creating_backup = True
+        harness.charm.reset_stale_connectivity_flag()
+        assert (
+            harness.get_relation_data(peer_rel_id, harness.charm.unit)["connectivity"] == "off"
+        )
+
+        # Already "on": no-op.
+        _patroni.return_value.is_creating_backup = False
+        with harness.hooks_disabled():
+            harness.update_relation_data(
+                peer_rel_id, harness.charm.unit.name, {"connectivity": "on"}
+            )
+        harness.charm.reset_stale_connectivity_flag()
+        assert (
+            harness.get_relation_data(peer_rel_id, harness.charm.unit)["connectivity"] == "on"
+        )
+
+
 def test_execute_command(harness):
     with (
         patch("backups.run") as _run,