Skip to content

reinit PostgreSQL only if there is a healthy majority #662

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
delgod wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

reinit PostgreSQL only if there is a healthy majority #662

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 22 additions & 2 deletions src/cluster.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -425,11 +425,16 @@ def is_creating_backup(self) -> bool:
for member in r.json()["members"]
)

def is_replication_healthy(self) -> bool:
def is_replication_healthy(self, majority_check: bool = False) -> bool:
"""Return whether the replication is healthy."""
expected_healthy_replicas_count = self.planned_units - 1
if majority_check:
expected_healthy_replicas_count = self.planned_units // 2
try:
for attempt in Retrying(stop=stop_after_delay(60), wait=wait_fixed(3)):
with attempt:
healthy_primary = False
healthy_replicas_count = 0
primary = self.get_primary()
primary_ip = self.get_member_ip(primary)
members_ips = {self.unit_ip}
Expand All @@ -447,7 +452,16 @@ def is_replication_healthy(self) -> bool:
logger.debug(
f"Failed replication check for {members_ip} with code {member_status.status_code}"
)
raise Exception
continue
if members_ip == primary_ip:
healthy_primary = True
else:
healthy_replicas_count += 1
if (
not healthy_primary
or healthy_replicas_count < expected_healthy_replicas_count
):
raise Exception
except RetryError:
logger.exception("replication is not healthy")
return False
Expand Down Expand Up @@ -816,6 +830,12 @@ def restart_postgresql(self) -> None:
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10))
def reinitialize_postgresql(self) -> None:
"""Reinitialize PostgreSQL."""

if not self.is_replication_healthy(majority_check=True):
logger.debug("skipping reinitialize PostgreSQL, because of lack of healthy majority")
raise Exception

logger.debug("reinitialize PostgreSQL")
requests.post(
f"{self._patroni_url}/reinitialize",
verify=self.verify,
Expand Down