re-enable remote-upgrade tests (#355)

* re-enable-tests

* remove skips in test_upgrades.py

* fix: upgrades fixes

* fix: remove debug

* fixes for network cut tests

* add fix to backup tests

* increase timeout yolo

* fix password tests

* Update test_upgrades.py

* remote apps do not need a resource

* make sure tests wait

* fixes for how app is passed through functions

* update app name in tests

---------

Co-authored-by: Neha Oudin <[email protected]>
Co-authored-by: Mehdi Bendriss <[email protected]>

Author: 3 people

tests/integration/backup_tests/test_backups.py

@@ -12,7 +12,13 @@
 import pytest_asyncio
 import yaml
 from pytest_operator.plugin import OpsTest
-from tenacity import RetryError, Retrying, stop_after_delay, wait_fixed
+from tenacity import (
+    RetryError,
+    Retrying,
+    stop_after_attempt,
+    stop_after_delay,
+    wait_fixed,
+)
 
 from ..ha_tests import helpers as ha_helpers
 from ..helpers import (
@@ -442,14 +448,13 @@ async def test_restore_new_cluster(
     ), "Backups from old cluster are listed as failed"
 
     # find most recent backup id and restore
-    action = await leader_unit.run_action(action_name="list-backups")
-    list_result = await action.wait()
-    list_result = list_result.results["backups"]
-    most_recent_backup = list_result.split("\n")[-1]
-    backup_id = most_recent_backup.split()[0]
-    action = await leader_unit.run_action(action_name="restore", **{"backup-id": backup_id})
-    restore = await action.wait()
-    assert restore.results["restore-status"] == "restore started", "restore not successful"
+    for attempt in Retrying(stop=stop_after_attempt(120), wait=wait_fixed(1), reraise=True):
+        action = await leader_unit.run_action(action_name="list-backups")
+        list_result = await action.wait()
+        list_result = list_result.results["backups"]
+        most_recent_backup = list_result.split("\n")[-1]
+        backup_id = most_recent_backup.split()[0]
+        assert "-----" not in backup_id, "list of backups are empty."
 
     # verify all writes are present
     try:

tests/integration/ha_tests/helpers.py

@@ -387,6 +387,7 @@ async def get_direct_mongo_client(
 ) -> MongoClient:
     """Returns a direct mongodb client potentially passing over some of the units."""
     port = MONGOS_PORT if mongos else MONGOD_PORT
+    mongodb_name = app_name or await get_application_name(ops_test, APP_NAME)
     if unit:
         url = await mongodb_uri(
             ops_test,
@@ -395,11 +396,10 @@ async def get_direct_mongo_client(
             port=port,
             username=username,
             password=password,
+            app_name=mongodb_name,
         )
         return MongoClient(url, directConnection=True)
 
-    mongodb_name = app_name or await get_application_name(ops_test, APP_NAME)
-
     for unit in ops_test.model.applications[mongodb_name].units:
         if unit.name not in excluded and unit.workload_status == "active":
             url = await mongodb_uri(
@@ -415,10 +415,10 @@ async def get_direct_mongo_client(
     assert False, "No fitting unit could be found"
 
 
-async def find_unit(ops_test: OpsTest, leader: bool) -> ops.model.Unit:
+async def find_unit(ops_test: OpsTest, leader: bool, app_name: str = APP_NAME) -> ops.model.Unit:
     """Helper function identifies a unit, based on need for leader or non-leader."""
     ret_unit = None
-    app = await get_application_name(ops_test, APP_NAME)
+    app = await get_application_name(ops_test, app_name)
     for unit in ops_test.model.applications[app].units:
         if await unit.is_leader_from_status() == leader:
             ret_unit = unit
@@ -666,11 +666,15 @@ def remove_instance_isolation(ops_test: OpsTest) -> None:
     reraise=True,
 )
 async def wait_until_unit_in_status(
-    ops_test: OpsTest, unit_to_check: Unit, online_unit: Unit, status: str
+    ops_test: OpsTest,
+    unit_to_check: Unit,
+    online_unit: Unit,
+    status: str,
+    app_name: str = APP_NAME,
 ) -> None:
     """Waits until a replica is in the provided status as reported by MongoDB or timeout occurs."""
     with await get_direct_mongo_client(
-        ops_test, online_unit.name, use_subprocess_to_get_password=True
+        ops_test, online_unit.name, use_subprocess_to_get_password=True, app_name=app_name
     ) as client:
         data = client.admin.command("replSetGetStatus")
 
@@ -972,5 +976,5 @@ async def count_writes(ops_test: OpsTest, app_name: str = None) -> int:
     """New versions of pymongo no longer support the count operation, instead find is used."""
     app_name = app_name or await get_app_name(ops_test)
     unit = ops_test.model.applications[app_name].units[0]
-    with await get_direct_mongo_client(ops_test, unit.name) as client:
+    with await get_direct_mongo_client(ops_test, unit.name, app_name=app_name) as client:
         return client[TEST_DB][TEST_COLLECTION].count_documents({})

tests/integration/sharding_tests/helpers.py

@@ -57,7 +57,11 @@ async def deploy_cluster_components(
     else:
         my_charm = MONGODB_CHARM_NAME
 
-    resources = {"mongodb-image": METADATA["resources"]["mongodb-image"]["upstream-source"]}
+    resources = (
+        None
+        if channel
+        else {"mongodb-image": METADATA["resources"]["mongodb-image"]["upstream-source"]}
+    )
     await ops_test.model.deploy(
         my_charm,
         resources=resources,

tests/integration/upgrades/helpers.py

@@ -24,7 +24,8 @@ async def assert_successful_run_upgrade_sequence(
     logger.info(f"Upgrading {app_name}")
 
     await ops_test.model.applications[app_name].refresh(path=new_charm)
-    await ops_test.model.wait_for_idle(apps=[app_name], timeout=1000, idle_period=30)
+    # TODO future work, resolve flickering status of app
+    await ops_test.model.wait_for_idle(apps=[app_name], timeout=1000, idle_period=90)
 
     # resume upgrade only needs to be ran when:
     # 1. there are more than one units in the application

tests/integration/upgrades/test_sharding_upgrades.py

@@ -71,9 +71,14 @@ async def add_writes_to_shards(ops_test: OpsTest):
 @pytest.mark.abort_on_fail
 async def test_build_and_deploy(ops_test: OpsTest) -> None:
     """Build deploy, and integrate, a sharded cluster."""
+    num_units_cluster_config = {
+        CONFIG_SERVER_APP_NAME: 3,
+        SHARD_ONE_APP_NAME: 3,
+        SHARD_TWO_APP_NAME: 3,
+    }
     await deploy_and_scale_application(ops_test)
 
-    await deploy_cluster_components(ops_test)
+    await deploy_cluster_components(ops_test, num_units_cluster_config, channel="6/edge")
 
     await ops_test.model.wait_for_idle(
         apps=CLUSTER_COMPONENTS,
@@ -94,7 +99,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
     await ops_test.model.applications[WRITE_APP].set_config({"mongos-uri": mongos_uri})
 
 
-@pytest.mark.skip()
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_upgrade(ops_test: OpsTest, add_writes_to_shards) -> None:
@@ -106,6 +110,14 @@ async def test_upgrade(ops_test: OpsTest, add_writes_to_shards) -> None:
             ops_test, sharding_component, new_charm=new_charm
         )
 
+    await ops_test.model.wait_for_idle(
+        apps=CLUSTER_COMPONENTS,
+        status="active",
+        timeout=1000,
+        idle_period=30,
+        raise_on_error=False,
+    )
+
     application_unit = ops_test.model.applications[WRITE_APP].units[0]
     stop_writes_action = await application_unit.run_action(
         "stop-continuous-writes",
@@ -139,44 +151,66 @@ async def test_upgrade(ops_test: OpsTest, add_writes_to_shards) -> None:
     ), "missed writes during upgrade procedure."
 
 
-@pytest.mark.skip()
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_pre_upgrade_check_success(ops_test: OpsTest) -> None:
     """Verify that the pre-refresh check succeeds in the happy path."""
+    await ops_test.model.wait_for_idle(
+        apps=CLUSTER_COMPONENTS,
+        status="active",
+        timeout=1000,
+        idle_period=30,
+        raise_on_error=False,
+    )
+
     for sharding_component in CLUSTER_COMPONENTS:
         leader_unit = await backup_helpers.get_leader_unit(ops_test, sharding_component)
         action = await leader_unit.run_action("pre-refresh-check")
         await action.wait()
         assert action.status == "completed", "pre-refresh-check failed, expected to succeed."
 
 
-@pytest.mark.skip()
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_pre_upgrade_check_failure(ops_test: OpsTest, chaos_mesh) -> None:
     """Verify that the pre-refresh check fails if there is a problem with one of the shards."""
+    await ops_test.model.wait_for_idle(
+        apps=CLUSTER_COMPONENTS,
+        status="active",
+        timeout=1000,
+        idle_period=30,
+        raise_on_error=False,
+    )
+
     leader_unit = await backup_helpers.get_leader_unit(ops_test, SHARD_TWO_APP_NAME)
 
     non_leader_unit = None
     for unit in ops_test.model.applications[SHARD_TWO_APP_NAME].units:
-        if unit != leader_unit:
+        if unit.name != leader_unit.name:
             non_leader_unit = unit
             break
 
     isolate_instance_from_cluster(ops_test, non_leader_unit.name)
     await wait_until_unit_in_status(
-        ops_test, non_leader_unit, leader_unit, "(not reachable/healthy)"
+        ops_test,
+        non_leader_unit,
+        leader_unit,
+        "(not reachable/healthy)",
+        app_name=SHARD_TWO_APP_NAME,
     )
 
     for sharding_component in CLUSTER_COMPONENTS:
         leader_unit = await backup_helpers.get_leader_unit(ops_test, sharding_component)
         action = await leader_unit.run_action("pre-refresh-check")
         await action.wait()
-        assert action.status == "completed", "pre-refresh-check failed, expected to succeed."
+        assert action.status == "failed", "pre-refresh-check succeeded, expected to fail."
 
     # restore network after test
     remove_instance_isolation(ops_test)
     await ops_test.model.wait_for_idle(
-        apps=[SHARD_TWO_APP_NAME], status="active", timeout=1000, idle_period=30
+        apps=[SHARD_TWO_APP_NAME],
+        status="active",
+        timeout=1000,
+        idle_period=30,
+        raise_on_error=False,
     )

tests/integration/upgrades/test_upgrades.py

@@ -9,6 +9,7 @@
 from pytest_operator.plugin import OpsTest
 
 from ..backup_tests import helpers as backup_helpers
+from ..ha_tests import helpers as ha_helpers
 from ..ha_tests.helpers import (
     count_writes,
     deploy_and_scale_application,
@@ -18,7 +19,7 @@
     remove_instance_isolation,
     wait_until_unit_in_status,
 )
-from ..helpers import check_or_scale_app, get_app_name, get_password, set_password
+from ..helpers import check_or_scale_app, get_app_name, get_password
 from .helpers import assert_successful_run_upgrade_sequence
 
 logger = logging.getLogger(__name__)
@@ -27,7 +28,15 @@
 MONGODB_CHARM_NAME = "mongodb-k8s"
 
 
-@pytest.mark.skip("skip until upgrades work has been released to charmhub")
+@pytest.fixture(scope="module")
+def chaos_mesh(ops_test: OpsTest) -> None:
+    ha_helpers.deploy_chaos_mesh(ops_test.model.info.name)
+
+    yield
+
+    ha_helpers.destroy_chaos_mesh(ops_test.model.info.name)
+
+
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_build_and_deploy(ops_test: OpsTest):
@@ -37,10 +46,10 @@ async def test_build_and_deploy(ops_test: OpsTest):
     db_app_name = await get_app_name(ops_test)
 
     if db_app_name:
-        await check_or_scale_app(ops_test, db_app_name, required_units=2)
+        await check_or_scale_app(ops_test, db_app_name, required_units=3)
         return
     else:
-        await ops_test.model.deploy(MONGODB_CHARM_NAME, channel="6/edge", num_units=2)
+        await ops_test.model.deploy(MONGODB_CHARM_NAME, channel="6/edge", num_units=3, trust=True)
 
     db_app_name = await get_app_name(ops_test)
     await ops_test.model.wait_for_idle(
@@ -50,7 +59,6 @@ async def test_build_and_deploy(ops_test: OpsTest):
     await relate_mongodb_and_application(ops_test, db_app_name, WRITE_APP)
 
 
-@pytest.mark.skip("skip until upgrades work has been released to charmhub")
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_successful_upgrade(ops_test: OpsTest, continuous_writes) -> None:
@@ -69,7 +77,6 @@ async def test_successful_upgrade(ops_test: OpsTest, continuous_writes) -> None:
     assert total_expected_writes == actual_writes, "missed writes during upgrade procedure."
 
 
-@pytest.mark.skip("skip until upgrades work has been released to charmhub")
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_preflight_check(ops_test: OpsTest) -> None:
@@ -82,7 +89,6 @@ async def test_preflight_check(ops_test: OpsTest) -> None:
     assert action.status == "completed", "pre-refresh-check failed, expected to succeed."
 
 
-@pytest.mark.skip("skip until upgrades work has been released to charmhub")
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_preflight_check_failure(ops_test: OpsTest, chaos_mesh) -> None:
@@ -91,7 +97,7 @@ async def test_preflight_check_failure(ops_test: OpsTest, chaos_mesh) -> None:
 
     non_leader_unit = None
     for unit in ops_test.model.applications[db_app_name].units:
-        if unit != leader_unit:
+        if unit.name != leader_unit.name:
             non_leader_unit = unit
             break
 
@@ -103,29 +109,31 @@ async def test_preflight_check_failure(ops_test: OpsTest, chaos_mesh) -> None:
     logger.info("Calling pre-refresh-check")
     action = await leader_unit.run_action("pre-refresh-check")
     await action.wait()
-    assert action.status == "completed", "pre-refresh-check failed, expected to succeed."
+    assert action.status == "failed", "pre-refresh-check succeeded, expected to fail."
 
     # restore network after test
     remove_instance_isolation(ops_test)
     await ops_test.model.wait_for_idle(
-        apps=[db_app_name], status="active", timeout=1000, idle_period=30
+        apps=[db_app_name], status="active", timeout=1000, idle_period=30, raise_on_error=False
     )
 
 
-@pytest.mark.skip("Missing upgrade code for now")
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_upgrade_password_change_fail(ops_test: OpsTest):
     app_name = await get_app_name(ops_test)
-    leader_id = await find_unit(ops_test, leader=True, app_name=app_name)
-
-    current_password = await get_password(ops_test, leader_id, app_name=app_name)
+    leader = await find_unit(ops_test, leader=True, app_name="mongodb-k8s")
+    leader_id = leader.name.split("/")[1]
+    current_password = await get_password(ops_test, leader_id, app_name="mongodb-k8s")
     new_charm = await ops_test.build_charm(".")
+
     await ops_test.model.applications[app_name].refresh(path=new_charm)
-    results = await set_password(ops_test, leader_id, password="0xdeadbeef", app_name=app_name)
 
-    assert results == "Cannot set passwords while an upgrade is in progress."
+    action = await ops_test.model.units.get(f"{app_name}/{leader_id}").run_action(
+        "set-password", **{"username": "username", "password": "new-password"}
+    )
+    action = await action.wait()
 
+    assert "Cannot set passwords while an upgrade is in progress." == action.message
     after_action_password = await get_password(ops_test, leader_id, app_name=app_name)
-
     assert current_password == after_action_password