[DPE-7502] Improve upgrade tests to support both (in)compatible scenarios (#955)

*Issue*:

The newly merged PR (with snap bump) has passed PR testing[1] but failing on release[2].
The reason here is a test imperfection (we've missed the on-Git case):

on-PR: we are testing incompatible case (due to +dirty charm.py version)
on-Git: we are testing compatible case, paused after the first unit upgrade
[1] #943
[2] https://github.com/canonical/postgresql-operator/actions/runs/15539934165/job/43748494710

*Solution*:
Improve the test to support both cases: on-PR and on-Git for already merged PRs.

Checklist
[x] I have added or updated any relevant documentation.
[x] I have cleaned any remaining cloud resources from my accounts.

Author: taurus-forever

tests/integration/ha_tests/test_upgrade.py

@@ -49,6 +49,7 @@ async def test_deploy_latest(ops_test: OpsTest) -> None:
         APPLICATION_NAME,
         num_units=1,
         channel="latest/edge",
+        config={"sleep_interval": 500},
     )
     logger.info("Wait for applications to become active")
     async with ops_test.fast_forward():
@@ -90,40 +91,43 @@ async def test_upgrade_from_edge(ops_test: OpsTest, continuous_writes, charm) ->
 
     logger.info("Wait for upgrade to start")
     try:
-        # Blocked status is expected due to compatibility checks.
+        # Blocked status is expected due to:
+        # (on PR) compatibility checks (on PR charm revision is '16/1.25.0+dirty...')
+        # (non-PR) the first unit upgraded and paused (pause_after_unit_refresh=first)
         await ops_test.model.block_until(lambda: application.status == "blocked", timeout=60 * 3)
 
-        logger.info("Wait for refresh to block due to compatibility checks")
+        logger.info("Wait for refresh to block as paused or incompatible")
         async with ops_test.fast_forward("60s"):
             await ops_test.model.wait_for_idle(
                 apps=[DATABASE_APP_NAME], idle_period=30, timeout=TIMEOUT
             )
 
-        assert "Refresh incompatible" in application.status_message, (
-            "Application refresh not blocked due to incompatibility"
-        )
-
         # Highest to lowest unit number
         refresh_order = sorted(
             application.units, key=lambda unit: int(unit.name.split("/")[1]), reverse=True
         )
-        action = await refresh_order[0].run_action(
-            "force-refresh-start", **{"check-compatibility": False}
-        )
-        await action.wait()
 
-        logger.info("Wait for first unit to upgrade")
-        async with ops_test.fast_forward("60s"):
-            await ops_test.model.wait_for_idle(
-                apps=[DATABASE_APP_NAME], idle_period=30, timeout=TIMEOUT
+        if "Refresh incompatible" in application.status_message:
+            logger.info("Application refresh is blocked due to incompatibility")
+
+            action = await refresh_order[0].run_action(
+                "force-refresh-start", **{"check-compatibility": False}
             )
+            await action.wait()
+
+            logger.info("Wait for first incompatible unit to upgrade")
+            async with ops_test.fast_forward("60s"):
+                await ops_test.model.wait_for_idle(
+                    apps=[DATABASE_APP_NAME], idle_period=30, timeout=TIMEOUT
+                )
 
         logger.info("Run resume-refresh action")
         action = await refresh_order[1].run_action("resume-refresh")
         await action.wait()
     except TimeoutError:
         # If the application didn't get into the blocked state, it should have upgraded only
         # the charm code because the snap revision didn't change.
+        logger.info("Upgrade completed without snap refresh (charm.py upgrade only)")
         assert application.status == "active", (
             "Application didn't reach blocked or active state after refresh attempt"
         )

tests/integration/ha_tests/test_upgrade_from_stable.py

@@ -41,6 +41,7 @@ async def test_deploy_stable(ops_test: OpsTest) -> None:
         APPLICATION_NAME,
         num_units=1,
         channel="latest/edge",
+        config={"sleep_interval": 500},
     )
     logger.info("Wait for applications to become active")
     async with ops_test.fast_forward():
@@ -83,40 +84,43 @@ async def test_upgrade_from_stable(ops_test: OpsTest, charm):
 
     logger.info("Wait for upgrade to start")
     try:
-        # Blocked status is expected due to compatibility checks.
+        # Blocked status is expected due to:
+        # (on PR) compatibility checks (on PR charm revision is '16/1.25.0+dirty...')
+        # (non-PR) the first unit upgraded and paused (pause_after_unit_refresh=first)
         await ops_test.model.block_until(lambda: application.status == "blocked", timeout=60 * 3)
 
-        logger.info("Wait for refresh to block due to compatibility checks")
+        logger.info("Wait for refresh to block as paused or incompatible")
         async with ops_test.fast_forward("60s"):
             await ops_test.model.wait_for_idle(
                 apps=[DATABASE_APP_NAME], idle_period=30, timeout=TIMEOUT
             )
 
-        assert "Refresh incompatible" in application.status_message, (
-            "Application refresh not blocked due to incompatibility"
-        )
-
         # Highest to lowest unit number
         refresh_order = sorted(
             application.units, key=lambda unit: int(unit.name.split("/")[1]), reverse=True
         )
-        action = await refresh_order[0].run_action(
-            "force-refresh-start", **{"check-compatibility": False}
-        )
-        await action.wait()
 
-        logger.info("Wait for first unit to upgrade")
-        async with ops_test.fast_forward("60s"):
-            await ops_test.model.wait_for_idle(
-                apps=[DATABASE_APP_NAME], idle_period=30, timeout=TIMEOUT
+        if "Refresh incompatible" in application.status_message:
+            logger.info("Application refresh is blocked due to incompatibility")
+
+            action = await refresh_order[0].run_action(
+                "force-refresh-start", **{"check-compatibility": False}
             )
+            await action.wait()
+
+            logger.info("Wait for first incompatible unit to upgrade")
+            async with ops_test.fast_forward("60s"):
+                await ops_test.model.wait_for_idle(
+                    apps=[DATABASE_APP_NAME], idle_period=30, timeout=TIMEOUT
+                )
 
         logger.info("Run resume-refresh action")
         action = await refresh_order[1].run_action("resume-refresh")
         await action.wait()
     except TimeoutError:
         # If the application didn't get into the blocked state, it should have upgraded only
         # the charm code because the snap revision didn't change.
+        logger.info("Upgrade completed without snap refresh (charm.py upgrade only)")
         assert application.status == "active", (
             "Application didn't reach blocked or active state after refresh attempt"
         )