[DPE-8462] Port tests (#1149)

* Upgrade tests WIP

* Missed helper and async tweaks

* Fix name and channel

* Copy conf test

* Leader resumes refresh

* K8s scale

* Wait for units to decrease

* More forceful force refresh

* Checking for blocked unit, not app

* Wait for unit status blocks

* Factor out refresh

* Retry resume action

* Wire switchover in async cluster upgrade

* Check for standby leader in replication health

Author: dragomirp

.gitignore

@@ -8,6 +8,7 @@ __pycache__/
 *.py[cod]
 ./requirements.txt
 requirements-last-build.txt
+.last_refresh_unit_status.json
 
 # PyCharm project folder.
 .idea/

poetry.lock

@@ -74,7 +74,7 @@ psycopg2-binary = "^2.9.10"
 boto3 = "*"
 tenacity = "^9.1.2"
 allure-pytest = "^2.15.0"
-jubilant = "^1.4.0"
+jubilant = "^1.5.0"
 tomli-w = "^1.2.0"
 tomli = "^2.3.0"
 
@@ -103,6 +103,7 @@ minversion = "6.0"
 log_cli_level = "INFO"
 asyncio_mode = "auto"
 markers = ["juju3", "juju_secrets"]
+addopts = "--exitfirst"
 
 # Linting tools configuration
 [tool.ruff]

pyproject.toml

@@ -2264,6 +2264,26 @@ def _can_connect_to_postgresql(self) -> bool:
             return False
         return True
 
+    def _api_update_config(self, available_cpu_cores: int) -> None:
+        # Use config value if set, calculate otherwise
+        if self.config.experimental_max_connections:
+            max_connections = self.config.experimental_max_connections
+        else:
+            max_connections = max(4 * available_cpu_cores, 100)
+
+        cfg_patch = {
+            "max_connections": max_connections,
+            "max_prepared_transactions": self.config.memory_max_prepared_transactions,
+            "max_replication_slots": 25,
+            "max_wal_senders": 25,
+            "shared_buffers": self.config.memory_shared_buffers,
+            "wal_keep_size": self.config.durability_wal_keep_size,
+        }
+        base_patch = {}
+        if primary_endpoint := self.async_replication.get_primary_cluster_endpoint():
+            base_patch["standby_cluster"] = {"host": primary_endpoint}
+        self._patroni.bulk_update_parameters_controller_by_patroni(cfg_patch, base_patch)
+
     def update_config(self, is_creating_backup: bool = False) -> bool:
         """Updates Patroni config file based on the existence of the TLS files."""
         # Retrieve PostgreSQL parameters.
@@ -2328,20 +2348,7 @@ def update_config(self, is_creating_backup: bool = False) -> bool:
             logger.debug("Early exit update_config: Patroni not started yet")
             return False
 
-        # Use config value if set, calculate otherwise
-        if self.config.experimental_max_connections:
-            max_connections = self.config.experimental_max_connections
-        else:
-            max_connections = max(4 * available_cpu_cores, 100)
-
-        self._patroni.bulk_update_parameters_controller_by_patroni({
-            "max_connections": max_connections,
-            "max_prepared_transactions": self.config.memory_max_prepared_transactions,
-            "max_replication_slots": 25,
-            "max_wal_senders": 25,
-            "shared_buffers": self.config.memory_shared_buffers,
-            "wal_keep_size": self.config.durability_wal_keep_size,
-        })
+        self._api_update_config(available_cpu_cores)
 
         self._patroni.ensure_slots_controller_by_patroni(replication_slots)
 

src/charm.py

@@ -389,7 +389,12 @@ def is_replication_healthy(self) -> bool:
         try:
             for attempt in Retrying(stop=stop_after_delay(60), wait=wait_fixed(3)):
                 with attempt:
-                    if not (primary := self.get_primary()):
+                    primary = (
+                        self.get_standby_leader()
+                        if self._charm.async_replication.get_primary_cluster_endpoint()
+                        else self.get_primary()
+                    )
+                    if not primary:
                         logger.debug("Failed replication check no primary reported")
                         raise Exception
 
@@ -494,15 +499,26 @@ def member_streaming(self) -> bool:
         return response.get("replication_state") == "streaming"
 
     @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10))
-    def bulk_update_parameters_controller_by_patroni(self, parameters: dict[str, Any]) -> None:
+    def bulk_update_parameters_controller_by_patroni(
+        self, parameters: dict[str, Any], base_parameters: dict[str, Any] | None
+    ) -> None:
         """Update the value of a parameter controller by Patroni.
 
         For more information, check https://patroni.readthedocs.io/en/latest/patroni_configuration.html#postgresql-parameters-controlled-by-patroni.
         """
+        if not base_parameters:
+            base_parameters = {}
         requests.patch(
             f"{self._patroni_url}/config",
             verify=self._verify,
-            json={"postgresql": {"parameters": parameters}},
+            json={
+                "postgresql": {
+                    "remove_data_directory_on_rewind_failure": False,
+                    "remove_data_directory_on_diverged_timelines": False,
+                    "parameters": parameters,
+                },
+                **base_parameters,
+            },
             auth=self._patroni_auth,
             timeout=API_REQUEST_TIMEOUT,
         )
@@ -727,15 +743,20 @@ def restart_postgresql(self) -> None:
             timeout=API_REQUEST_TIMEOUT,
         )
 
-    def switchover(self, candidate: str | None = None, wait: bool = True) -> None:
+    def switchover(
+        self, candidate: str | None = None, wait: bool = True, async_cluster: bool = False
+    ) -> None:
         """Trigger a switchover."""
         # Try to trigger the switchover.
         if candidate is not None:
             candidate = candidate.replace("/", "-")
 
         for attempt in Retrying(stop=stop_after_delay(60), wait=wait_fixed(3)):
             with attempt:
-                primary = self.get_primary()
+                primary = self.get_primary() if not async_cluster else self.get_standby_leader()
+                if primary == candidate:
+                    logger.info("Candidate and leader are the same")
+                    return
                 r = requests.post(
                     f"{self._patroni_url}/switchover",
                     json={"leader": primary, "candidate": candidate},

src/patroni.py

@@ -65,7 +65,12 @@ def run_pre_refresh_checks_after_1_unit_refreshed(self) -> None:
             )
         else:
             try:
-                self._charm._patroni.switchover(candidate=last_unit_to_refresh)
+                self._charm._patroni.switchover(
+                    candidate=last_unit_to_refresh,
+                    async_cluster=bool(
+                        self._charm.async_replication.get_primary_cluster_endpoint()
+                    ),
+                )
             except SwitchoverFailedError as e:
                 logger.warning(f"switchover failed with reason: {e}")
                 raise charm_refresh.PrecheckFailed("Unable to switch primary") from None

src/refresh.py

@@ -104,28 +104,15 @@ def juju(request: pytest.FixtureRequest):
 
     This adds command line parameter ``--keep-models`` (see help for details).
     """
-    controller = request.config.getoption("--controller")
     model = request.config.getoption("--model")
-    controller_and_model = None
-    if controller and model:
-        controller_and_model = f"{controller}:{model}"
-    elif controller:
-        controller_and_model = controller
-    elif model:
-        controller_and_model = model
     keep_models = bool(request.config.getoption("--keep-models"))
 
-    if controller_and_model:
-        juju = jubilant.Juju(model=controller_and_model)  # type: ignore
+    if model:
+        juju = jubilant.Juju(model=model)
         yield juju
-        log = juju.debug_log(limit=1000)
     else:
         with jubilant.temp_model(keep=keep_models) as juju:
             yield juju
-            log = juju.debug_log(limit=1000)
-
-    if request.session.testsfailed:
-        print(log, end="")
 
 
 @pytest.fixture(scope="module")