CLOUDP-294373: Configure ClusterIP services for mongos (#4134)

# Summary

When configuring a multi-cluster sharded deployment with external access
but no external domain, the operator was not creating clusterIP services
for the components. They could not reach each other.

## Proof of Work

New test ensuring we can deploy a multi-cluster sharded resource with
`externalAccess` defined, but no `externalDomain`, with a service mesh.
The test was failing without the fix, as pods couldn't communicate.

Author: Julien-Ben

.evergreen-tasks.yml

@@ -1165,6 +1165,11 @@ tasks:
     commands:
       - func: e2e_test
 
+  - name: e2e_multi_cluster_sharded_external_access_no_ext_domain
+    tags: [ "patch-run" ]
+    commands:
+      - func: e2e_test
+
   - name: e2e_multi_cluster_sharded_tls_no_mesh
     tags: [ "patch-run" ]
     commands:

.evergreen.yml

@@ -789,6 +789,7 @@ task_groups:
       - e2e_multi_cluster_sharded_simplest
       - e2e_multi_cluster_sharded_snippets
       - e2e_multi_cluster_sharded_simplest_no_mesh
+      - e2e_multi_cluster_sharded_external_access_no_ext_domain
       - e2e_multi_cluster_sharded_tls_no_mesh
       - e2e_multi_cluster_sharded_tls
       - e2e_multi_cluster_sharded_disaster_recovery

RELEASE_NOTES.md

@@ -8,6 +8,7 @@
 ## Bug Fixes
 * Fixes the bug when status of `MongoDBUser` was being set to `Updated` prematurely. For example, new users were not immediately usable following `MongoDBUser` creation despite the operator reporting `Updated` state.
 * Fixed a bug causing cluster health check issues when ordering of users and tokens differed in Kubeconfig.
+* Fixed a bug when deploying a Multi-Cluster sharded resource with an external access configuration could result in pods not being able to reach each others.
 
 # MongoDB Enterprise Kubernetes Operator 1.31.0
 

controllers/operator/clusterchecks_test.go

@@ -137,7 +137,7 @@ func (c *clusterChecks) checkExternalServices(ctx context.Context, statefulSetNa
 	}
 }
 
-func (c *clusterChecks) checkExternalServicesDontNotExist(ctx context.Context, statefulSetName string, expectedMembers int) {
+func (c *clusterChecks) checkExternalServicesDontExist(ctx context.Context, statefulSetName string, expectedMembers int) {
 	for podIdx := 0; podIdx < expectedMembers; podIdx++ {
 		svc := corev1.Service{}
 		serviceName := fmt.Sprintf("%s-%d-svc-external", statefulSetName, podIdx)

controllers/operator/mongodbshardedcluster_controller.go

@@ -2374,22 +2374,6 @@ func (r *ShardedClusterReconcileHelper) GetConfigSrvServiceName(memberCluster mu
 	}
 }
 
-func (r *ShardedClusterReconcileHelper) GetMongosServiceName(memberCluster multicluster.MemberCluster) string {
-	if memberCluster.Legacy {
-		return r.sc.ServiceName()
-	} else {
-		return dns.GetMultiHeadlessServiceName(r.sc.Name, memberCluster.Index)
-	}
-}
-
-func (r *ShardedClusterReconcileHelper) GetShardServiceName(memberCluster multicluster.MemberCluster) string {
-	if memberCluster.Legacy {
-		return r.sc.ServiceName()
-	} else {
-		return r.sc.ShardServiceName()
-	}
-}
-
 func (r *ShardedClusterReconcileHelper) replicateAgentKeySecret(ctx context.Context, conn om.Connection, agentKey string, log *zap.SugaredLogger) error {
 	for _, memberCluster := range getHealthyMemberClusters(r.allMemberClusters) {
 		var databaseSecretPath string
@@ -2519,6 +2503,7 @@ func (r *ShardedClusterReconcileHelper) reconcileConfigServerServices(ctx contex
 		if configServerExternalAccess == nil {
 			configServerExternalAccess = r.sc.Spec.DbCommonSpec.ExternalAccessConfiguration
 		}
+		// Config servers need external services only if an externalDomain is configured
 		if configServerExternalAccess != nil && configServerExternalAccess.ExternalDomain != nil {
 			log.Debugf("creating external services for %s in cluster: %s", r.sc.ConfigRsName(), memberCluster.Name)
 			svc, err := r.getPodExternalService(memberCluster,
@@ -2532,7 +2517,9 @@ func (r *ShardedClusterReconcileHelper) reconcileConfigServerServices(ctx contex
 			if err = mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc); err != nil && !errors.IsAlreadyExists(err) {
 				return xerrors.Errorf("failed to create external service %s in cluster: %s, err: %w", svc.Name, memberCluster.Name, err)
 			}
-		} else {
+		}
+		// We don't need internal per-pod services in case we have externalAccess configured AND an external domain
+		if configServerExternalAccess == nil || configServerExternalAccess.ExternalDomain == nil {
 			log.Debugf("creating internal services for %s in cluster: %s", r.sc.ConfigRsName(), memberCluster.Name)
 			svc := r.getPodService(r.sc.ConfigRsName(), memberCluster, podNum, portOrDefault)
 			if err := mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc); err != nil && !errors.IsAlreadyExists(err) {
@@ -2556,6 +2543,7 @@ func (r *ShardedClusterReconcileHelper) reconcileShardServices(ctx context.Conte
 	scaler := r.GetShardScaler(shardIdx, memberCluster)
 
 	for podNum := 0; podNum < scaler.DesiredReplicas(); podNum++ {
+		// Shards need external services only if an externalDomain is configured
 		if shardsExternalAccess != nil && shardsExternalAccess.ExternalDomain != nil {
 			log.Debugf("creating external services for %s in cluster: %s", r.sc.ShardRsName(shardIdx), memberCluster.Name)
 			svc, err := r.getPodExternalService(
@@ -2570,7 +2558,9 @@ func (r *ShardedClusterReconcileHelper) reconcileShardServices(ctx context.Conte
 			if err = mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc); err != nil && !errors.IsAlreadyExists(err) {
 				return xerrors.Errorf("failed to create external service %s in cluster: %s, err: %w", svc.Name, memberCluster.Name, err)
 			}
-		} else {
+		}
+		// We don't need internal per-pod services in case we have externalAccess configured AND an external domain
+		if shardsExternalAccess == nil || shardsExternalAccess.ExternalDomain == nil {
 			log.Debugf("creating internal services for %s in cluster: %s", r.sc.ShardRsName(shardIdx), memberCluster.Name)
 			svc := r.getPodService(r.sc.ShardRsName(shardIdx), memberCluster, podNum, portOrDefault)
 			if err := mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc); err != nil {
@@ -2595,6 +2585,7 @@ func (r *ShardedClusterReconcileHelper) reconcileMongosServices(ctx context.Cont
 		if mongosExternalAccess == nil {
 			mongosExternalAccess = r.sc.Spec.DbCommonSpec.ExternalAccessConfiguration
 		}
+		// Mongos always need external services if externalAccess is configured
 		if mongosExternalAccess != nil {
 			log.Debugf("creating external services for %s in cluster: %s", r.sc.MongosRsName(), memberCluster.Name)
 			svc, err := r.getPodExternalService(memberCluster,
@@ -2608,7 +2599,9 @@ func (r *ShardedClusterReconcileHelper) reconcileMongosServices(ctx context.Cont
 			if err = mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc); err != nil && !errors.IsAlreadyExists(err) {
 				return xerrors.Errorf("failed to create external service %s in cluster: %s, err: %w", svc.Name, memberCluster.Name, err)
 			}
-		} else {
+		}
+		// We don't need internal per-pod services in case we have externalAccess configured AND an external domain
+		if mongosExternalAccess == nil || mongosExternalAccess.ExternalDomain == nil {
 			log.Debugf("creating internal services for %s in cluster: %s", r.sc.MongosRsName(), memberCluster.Name)
 			svc := r.getPodService(r.sc.MongosRsName(), memberCluster, podNum, portOrDefault)
 			if err := mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc); err != nil && !errors.IsAlreadyExists(err) {
@@ -2617,6 +2610,7 @@ func (r *ShardedClusterReconcileHelper) reconcileMongosServices(ctx context.Cont
 
 			_ = append(allServices, &svc)
 		}
+
 		if err := createHeadlessServiceForStatefulSet(ctx, r.sc.MongosRsName(), portOrDefault, r.sc.Namespace, memberCluster); err != nil {
 			return err
 		}

controllers/operator/mongodbshardedcluster_controller_multi_test.go

@@ -550,21 +550,22 @@ func TestReconcileCreateMultiClusterShardedClusterWithExternalAccessAndNoExterna
 		configSrvStsName := fmt.Sprintf("%s-config-%d", sc.Name, clusterIdx)
 		configMembers := memberClusters.ConfigServerDistribution[clusterSpecItem.ClusterName]
 		memberClusterChecks.checkInternalServices(ctx, configSrvStsName)
-		memberClusterChecks.checkExternalServicesDontNotExist(ctx, configSrvStsName, configMembers)
+		memberClusterChecks.checkExternalServicesDontExist(ctx, configSrvStsName, configMembers)
 		memberClusterChecks.checkPerPodServices(ctx, configSrvStsName, configMembers)
 
 		mongosStsName := fmt.Sprintf("%s-mongos-%d", sc.Name, clusterIdx)
 		mongosMembers := memberClusters.MongosDistribution[clusterSpecItem.ClusterName]
 		memberClusterChecks.checkExternalServices(ctx, mongosStsName, mongosMembers)
 		memberClusterChecks.checkInternalServices(ctx, mongosStsName)
-		memberClusterChecks.checkPerPodServicesDontExist(ctx, mongosStsName, mongosMembers)
+		// Without external domain, we need per-pod mongos services
+		memberClusterChecks.checkPerPodServices(ctx, mongosStsName, mongosMembers)
 		memberClusterChecks.checkServiceAnnotations(ctx, mongosStsName, mongosMembers, sc, clusterSpecItem.ClusterName, clusterIdx, test.ExampleAccessWithNoExternalDomain.MongosExternalDomain)
 
 		for shardIdx := 0; shardIdx < memberClusters.ShardCount(); shardIdx++ {
 			shardStsName := fmt.Sprintf("%s-%d-%d", sc.Name, shardIdx, clusterIdx)
 			shardMembers := memberClusters.ShardDistribution[shardIdx][clusterSpecItem.ClusterName]
 			memberClusterChecks.checkInternalServices(ctx, shardStsName)
-			memberClusterChecks.checkExternalServicesDontNotExist(ctx, shardStsName, shardMembers)
+			memberClusterChecks.checkExternalServicesDontExist(ctx, shardStsName, shardMembers)
 			memberClusterChecks.checkPerPodServices(ctx, shardStsName, shardMembers)
 		}
 		memberClusterChecks.checkHostnameOverrideConfigMap(ctx, fmt.Sprintf("%s-hostname-override", sc.Name), expectedHostnameOverrideMap)

controllers/operator/mongodbshardedcluster_controller_test.go

@@ -149,16 +149,16 @@ func TestReconcileCreateSingleClusterShardedClusterWithExternalDomainSimplest(t
 	memberClusterChecks.checkServiceAnnotations(ctx, mongosStatefulSetName, sc.Spec.MongosCount, sc, multicluster.LegacyCentralClusterName, 0, test.ExampleExternalClusterDomains.SingleClusterDomain)
 
 	configServerStatefulSetName := fmt.Sprintf("%s-config", sc.Name)
-	memberClusterChecks.checkExternalServicesDontNotExist(ctx, configServerStatefulSetName, sc.Spec.ConfigServerCount)
+	memberClusterChecks.checkExternalServicesDontExist(ctx, configServerStatefulSetName, sc.Spec.ConfigServerCount)
 	memberClusterChecks.checkPerPodServicesDontExist(ctx, configServerStatefulSetName, sc.Spec.ConfigServerCount)
 	// This is something to be unified - why MC and SC Services are called differently?
 	configServerInternalServiceName := fmt.Sprintf("%s-cs", sc.Name)
 	memberClusterChecks.checkServiceExists(ctx, configServerInternalServiceName)
 
-	memberClusterChecks.checkExternalServicesDontNotExist(ctx, fmt.Sprintf("%s-config", sc.Name), sc.Spec.ConfigServerCount)
+	memberClusterChecks.checkExternalServicesDontExist(ctx, fmt.Sprintf("%s-config", sc.Name), sc.Spec.ConfigServerCount)
 	for shardIdx := 0; shardIdx < sc.Spec.ShardCount; shardIdx++ {
 		shardStatefulSetName := fmt.Sprintf("%s-%d", sc.Name, shardIdx)
-		memberClusterChecks.checkExternalServicesDontNotExist(ctx, shardStatefulSetName, sc.Spec.ShardCount)
+		memberClusterChecks.checkExternalServicesDontExist(ctx, shardStatefulSetName, sc.Spec.ShardCount)
 		memberClusterChecks.checkPerPodServicesDontExist(ctx, shardStatefulSetName, sc.Spec.ShardCount)
 		// This is something to be unified - why MC and SC Services are called differently?
 		shardInternalServiceName := fmt.Sprintf("%s-sh", sc.Name)

docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/e2e_multi_cluster_sharded_external_access_no_ext_domain.py

@@ -0,0 +1,113 @@
+from collections import defaultdict
+from typing import Dict, List, Optional
+
+import kubernetes
+from kubernetes import client
+from kubetester import find_fixture, try_load
+from kubetester.kubetester import ensure_ent_version
+from kubetester.mongodb import MongoDB, Phase
+from kubetester.operator import Operator
+from pytest import fixture, mark
+from tests import test_logger
+from tests.conftest import get_member_cluster_api_client, get_member_cluster_names
+from tests.shardedcluster.conftest import (
+    enable_multi_cluster_deployment,
+    setup_external_access,
+)
+
+MDB_RESOURCE_NAME = "sh"
+logger = test_logger.get_test_logger(__name__)
+
+
+@fixture(scope="module")
+def sharded_cluster(namespace: str, custom_mdb_version: str) -> MongoDB:
+    resource = MongoDB.from_yaml(
+        find_fixture("sharded-cluster-multi-cluster.yaml"), namespace=namespace, name=MDB_RESOURCE_NAME
+    )
+
+    if try_load(resource):
+        return resource
+
+    resource.set_version(ensure_ent_version(custom_mdb_version))
+
+    enable_multi_cluster_deployment(resource=resource)
+    setup_external_access(resource=resource, enable_external_domain=False)
+
+    resource.set_architecture_annotation()
+
+    return resource
+
+
+@mark.e2e_multi_cluster_sharded_external_access_no_ext_domain
+def test_deploy_operator(multi_cluster_operator: Operator):
+    multi_cluster_operator.assert_is_running()
+
+
+@mark.e2e_multi_cluster_sharded_external_access_no_ext_domain
+def test_sharded_cluster(sharded_cluster: MongoDB):
+    sharded_cluster.update()
+    sharded_cluster.assert_reaches_phase(Phase.Running, timeout=800)
+
+
+def service_exists(service_name: str, namespace: str, api_client: Optional[kubernetes.client.ApiClient] = None) -> bool:
+    try:
+        client.CoreV1Api(api_client=api_client).read_namespaced_service(service_name, namespace)
+    except client.rest.ApiException as e:
+        logger.error(f"Error reading {service_name}: {e}")
+        return False
+    return True
+
+
+@mark.e2e_multi_cluster_sharded_external_access_no_ext_domain
+def test_services_were_created(sharded_cluster: MongoDB, namespace: str):
+    resource_name = sharded_cluster.name
+    expected_services: Dict[str, List[str]] = defaultdict(list)
+    member_clusters = get_member_cluster_names()
+
+    # Global services
+    for cluster in member_clusters:
+        expected_services[cluster].append(f"{resource_name}-svc")
+        expected_services[cluster].append(f"{resource_name}-{resource_name}")
+
+    # All components get a headless service and per-pod services
+    # Config server also gets an additional headless service suffixed -cs
+    config_clusters = sharded_cluster["spec"]["configSrv"]["clusterSpecList"]
+    for idx, cluster_spec in enumerate(config_clusters):
+        members = cluster_spec["members"]
+        expected_services[cluster_spec["clusterName"]].append(f"{resource_name}-config-{idx}-svc")
+        expected_services[cluster_spec["clusterName"]].append(f"{resource_name}-{idx}-cs")
+        for pod in range(members):
+            expected_services[cluster_spec["clusterName"]].append(f"{resource_name}-config-{idx}-{pod}-svc")
+
+    # Mongos also get an external service per pod
+    mongos_clusters = sharded_cluster["spec"]["mongos"]["clusterSpecList"]
+    for idx, cluster_spec in enumerate(mongos_clusters):
+        members = cluster_spec["members"]
+        cluster_name = cluster_spec["clusterName"]
+        expected_services[cluster_name].append(f"{resource_name}-mongos-{idx}-svc")
+        for pod in range(members):
+            expected_services[cluster_name].append(f"{resource_name}-mongos-{idx}-{pod}-svc")
+            expected_services[cluster_name].append(f"{resource_name}-mongos-{idx}-{pod}-svc-external")
+
+    shard_count = sharded_cluster["spec"]["shardCount"]
+    shard_clusters = sharded_cluster["spec"]["shard"]["clusterSpecList"]
+    for shard in range(shard_count):
+        for idx, cluster_spec in enumerate(shard_clusters):
+            members = cluster_spec["members"]
+            cluster_name = cluster_spec["clusterName"]
+            expected_services[cluster_name].append(f"{resource_name}-{shard}-{idx}-svc")
+            for pod in range(members):
+                expected_services[cluster_name].append(f"{resource_name}-{shard}-{idx}-{pod}-svc")
+
+    logger.debug("Asserting the following services exist:")
+    for cluster, services in expected_services.items():
+        logger.debug(f"Cluster: {cluster}, service count: {len(services)}")
+        logger.debug(f"Services: {services}")
+
+    # Assert that each expected service exists in its corresponding cluster.
+    for cluster, services in expected_services.items():
+        api_client = get_member_cluster_api_client(cluster)  # Retrieve the API client for the cluster
+        for svc in services:
+            assert service_exists(
+                svc, namespace, api_client
+            ), f"Service {svc} not found. Cluster: {cluster} Namespace: {namespace}"

docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_simplest_no_mesh.py

@@ -28,7 +28,7 @@ def sharded_cluster(namespace: str, custom_mdb_version: str) -> MongoDB:
     resource.set_version(ensure_ent_version(custom_mdb_version))
 
     enable_multi_cluster_deployment(resource=resource)
-    setup_external_access(resource=resource)
+    setup_external_access(resource=resource, enable_external_domain=True)
 
     resource.set_architecture_annotation()
 
@@ -55,7 +55,7 @@ def test_deploy_operator(multi_cluster_operator: Operator):
 @mark.e2e_multi_cluster_sharded_simplest_no_mesh
 def test_sharded_cluster(sharded_cluster: MongoDB):
     sharded_cluster.update()
-    sharded_cluster.assert_reaches_phase(Phase.Running, timeout=1800)
+    sharded_cluster.assert_reaches_phase(Phase.Running, timeout=800)
 
 
 # Testing connectivity with External Access requires using the same DNS as deployed in Kube within

docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_tls_no_mesh.py

@@ -152,7 +152,7 @@ def test_deploy_certs(all_certs, agent_certs):
 @mark.e2e_multi_cluster_sharded_tls_no_mesh
 def test_sharded_cluster_with_prefix_gets_to_running_state(sharded_cluster: MongoDB):
     sharded_cluster.update()
-    sharded_cluster.assert_reaches_phase(Phase.Running, timeout=1800)
+    sharded_cluster.assert_reaches_phase(Phase.Running, timeout=800)
 
 
 # TODO: (slaskawi) clearly the client tries to connect to mongos without TLS (we can see this in the logs).