Ensure the operator runs correctly after MEKO uninstallation (#65)

# Summary

Add a last step to the mck -> meko upgrade, to ensure the multi cluster
member list configmap is still present and correct after uninstalling
the helm release of MEKO.

Passing patch:
https://spruce.mongodb.com/version/68124020fead0f0007f00929/tasks?sorts=STATUS%3AASC%3BBASE_STATUS%3ADESC

Author: Julien-Ben

docker/mongodb-kubernetes-tests/tests/conftest.py

@@ -70,6 +70,7 @@
 DATABASE_SA_NAME = "mongodb-kubernetes-database-pods"
 OM_SA_NAME = "mongodb-kubernetes-ops-manager"
 TELEMETRY_CONFIGMAP_NAME = LEGACY_OPERATOR_NAME + "-telemetry"
+MULTI_CLUSTER_MEMBER_LIST_CONFIGMAP = LEGACY_OPERATOR_NAME + "-member-list"
 
 logger = test_logger.get_test_logger(__name__)
 

docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery.py

@@ -14,7 +14,11 @@
 from kubetester.mongodb import Phase
 from kubetester.opsmanager import MongoDBOpsManager
 from pytest import fixture, mark
-from tests.conftest import create_appdb_certs, get_member_cluster_api_client
+from tests.conftest import (
+    MULTI_CLUSTER_MEMBER_LIST_CONFIGMAP,
+    create_appdb_certs,
+    get_member_cluster_api_client,
+)
 from tests.multicluster.conftest import cluster_spec_list
 
 FAILED_MEMBER_CLUSTER_NAME = "kind-e2e-cluster-3"
@@ -121,7 +125,7 @@ def test_remove_cluster_from_operator_member_list_to_simulate_it_is_unhealthy(
 ):
     member_list_cm = read_configmap(
         namespace,
-        "mongodb-enterprise-operator-member-list",
+        MULTI_CLUSTER_MEMBER_LIST_CONFIGMAP,
         api_client=central_cluster_client,
     )
     # this if is only for allowing re-running the test locally
@@ -132,7 +136,7 @@ def test_remove_cluster_from_operator_member_list_to_simulate_it_is_unhealthy(
     # this will trigger operators restart as it panics on changing the configmap
     update_configmap(
         namespace,
-        "mongodb-enterprise-operator-member-list",
+        MULTI_CLUSTER_MEMBER_LIST_CONFIGMAP,
         member_list_cm,
         api_client=central_cluster_client,
     )

docker/mongodb-kubernetes-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_disaster_recovery.py

@@ -25,7 +25,11 @@
 from kubetester.opsmanager import MongoDBOpsManager
 from pytest import fixture, mark
 from tests import test_logger
-from tests.conftest import get_central_cluster_client, get_member_cluster_api_client
+from tests.conftest import (
+    MULTI_CLUSTER_MEMBER_LIST_CONFIGMAP,
+    get_central_cluster_client,
+    get_member_cluster_api_client,
+)
 from tests.multicluster.conftest import cluster_spec_list
 from tests.shardedcluster.conftest import (
     enable_multi_cluster_deployment,
@@ -157,7 +161,7 @@ def test_create_sharded_cluster(self, sc: MongoDB, config_version_store):
     def test_remove_cluster_from_operator_member_list_to_simulate_it_is_unhealthy(
         self, namespace, central_cluster_client: kubernetes.client.ApiClient, multi_cluster_operator: Operator
     ):
-        operator_cm_name = "mongodb-enterprise-operator-member-list"
+        operator_cm_name = MULTI_CLUSTER_MEMBER_LIST_CONFIGMAP
         logger.debug(f"Deleting cluster {FAILED_MEMBER_CLUSTER_NAME} from configmap {operator_cm_name}")
         member_list_cm = read_configmap(
             namespace,

docker/mongodb-kubernetes-tests/tests/upgrades/meko_mck_upgrade.py

@@ -15,12 +15,18 @@
 from tests.conftest import (
     LEGACY_MULTI_CLUSTER_OPERATOR_NAME,
     LEGACY_OPERATOR_NAME,
+    MULTI_CLUSTER_MEMBER_LIST_CONFIGMAP,
+    MULTI_CLUSTER_OPERATOR_NAME,
+    OPERATOR_NAME,
     get_multi_cluster_operator,
     is_multi_cluster,
     log_deployments_info,
     setup_log_rotate_for_agents,
 )
 from tests.multicluster.conftest import cluster_spec_list
+from tests.multicluster_appdb.multicluster_appdb_state_operator_upgrade_downgrade import (
+    assert_cm_expected_data,
+)
 from tests.upgrades import downscale_operator_deployment
 
 logger = test_logger.get_test_logger(__name__)
@@ -152,3 +158,26 @@ def test_replicaset_reconciled(replica_set: MongoDB):
 def test_uninstall_latest_official_operator(namespace: str):
     helm_uninstall("mongodb-enterprise-operator-multi-cluster" if is_multi_cluster() else "mongodb-enterprise-operator")
     log_deployments_info(namespace)
+
+
+@mark.e2e_meko_mck_upgrade
+def test_operator_still_running(namespace: str, central_cluster_client: client.ApiClient, member_cluster_names):
+    operator_name = MULTI_CLUSTER_OPERATOR_NAME if is_multi_cluster() else OPERATOR_NAME
+    operator_instance = Operator(
+        name=operator_name,
+        namespace=namespace,
+    )
+    logger.info(f"Checking status of operator '{operator_name}' in namespace '{namespace}'")
+    operator_instance.assert_is_running()
+    log_deployments_info(namespace)
+
+    if is_multi_cluster():
+        # Check if member-list configmap is present and content is correct
+        logger.info(f"Checking correctness of member list configmap")
+        expected_data = {name: "" for name in member_cluster_names}
+        assert_cm_expected_data(
+            name=MULTI_CLUSTER_MEMBER_LIST_CONFIGMAP,
+            namespace=namespace,
+            expected_data=expected_data,
+            central_cluster_client=central_cluster_client,
+        )