CLOUDP-111968: Update automation config when updating TLS (#905)

Author: mircea-cosbuc

.action_templates/jobs/tests.yaml

@@ -33,6 +33,8 @@ tests:
           distro: ubi
         - test-name: replica_set_tls_rotate
           distro: ubi
+        - test-name: replica_set_tls_rotate_delete_sts
+          distro: ubi
         - test-name: replica_set_tls_upgrade
           distro: ubi
         - test-name: statefulset_arbitrary_config

.github/workflows/e2e-fork.yml

@@ -115,6 +115,8 @@ jobs:
           distro: ubi
         - test-name: replica_set_tls_rotate
           distro: ubi
+        - test-name: replica_set_tls_rotate_delete_sts
+          distro: ubi
         - test-name: replica_set_tls_upgrade
           distro: ubi
         - test-name: statefulset_arbitrary_config

.github/workflows/e2e.yml

@@ -121,6 +121,8 @@ jobs:
           distro: ubi
         - test-name: replica_set_tls_rotate
           distro: ubi
+        - test-name: replica_set_tls_rotate_delete_sts
+          distro: ubi
         - test-name: replica_set_tls_upgrade
           distro: ubi
         - test-name: statefulset_arbitrary_config

controllers/replica_set_controller.go

@@ -396,7 +396,11 @@ func (r *ReplicaSetReconciler) deployAutomationConfig(mdb mdbv1.MongoDBCommunity
 // functions should be sequential or not. A value of false indicates they will run in reversed order.
 func (r *ReplicaSetReconciler) shouldRunInOrder(mdb mdbv1.MongoDBCommunity) bool {
 	// The only case when we push the StatefulSet first is when we are ensuring TLS for the already existing ReplicaSet
-	_, err := r.client.GetStatefulSet(mdb.NamespacedName())
+	sts, err := r.client.GetStatefulSet(mdb.NamespacedName())
+	if !statefulset.IsReady(sts, mdb.StatefulSetReplicasThisReconciliation()) && mdb.Spec.Security.TLS.Enabled {
+		r.log.Debug("Enabling TLS on a deployment with a StatefulSet that is not Ready, the Automation Config must be updated first")
+		return true
+	}
 	if err == nil && mdb.Spec.Security.TLS.Enabled {
 		r.log.Debug("Enabling TLS on an existing deployment, the StatefulSet must be updated first")
 		return false

deploy/e2e/role.yaml

@@ -134,6 +134,7 @@ rules:
   - watch
   - create
   - delete
+  - patch
   - update
 - apiGroups:
   - acme.cert-manager.io

test/e2e/replica_set_tls_rotate_delete_sts/replica_set_tls_rotate_delete_sts_test.go

@@ -0,0 +1,65 @@
+package replica_set_tls_rotate_delete_sts
+
+import (
+	"os"
+	"testing"
+
+	"fmt"
+
+	. "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester"
+
+	e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e"
+	"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests"
+	"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup"
+	"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/tlstests"
+)
+
+func TestMain(m *testing.M) {
+	code, err := e2eutil.RunTest(m)
+	if err != nil {
+		fmt.Println(err)
+	}
+	os.Exit(code)
+}
+
+func TestReplicaSetTLSRotateDeleteSts(t *testing.T) {
+	resourceName := "mdb-tls"
+
+	ctx, testConfig := setup.SetupWithTLS(t, resourceName)
+	defer ctx.Teardown()
+
+	mdb, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace)
+	mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false)
+
+	_, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tester, err := FromResource(t, mdb)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	clientCert, err := GetClientCert(mdb)
+	if err != nil {
+		t.Fatal(err)
+	}
+	initialCertSerialNumber := clientCert.SerialNumber
+
+	t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx))
+	t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb))
+	t.Run("Wait for TLS to be enabled", tester.HasTlsMode("requireSSL", 60, WithTls(mdb)))
+	t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(mdb)))
+	t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3, WithTls(mdb)))
+	t.Run("Test TLS required", tester.ConnectivityFails(WithoutTls()))
+
+	t.Run("MongoDB is reachable while certificate is rotated", func(t *testing.T) {
+		t.Run("Delete Statefulset", mongodbtests.DeleteStatefulSet(&mdb))
+		t.Run("Update certificate secret", tlstests.RotateCertificate(&mdb))
+		t.Run("Wait for certificate to be rotated", tester.WaitForRotatedCertificate(mdb, initialCertSerialNumber))
+		t.Run("Test Replica Set Recovers", mongodbtests.StatefulSetBecomesReady(&mdb))
+		t.Run("Wait for MongoDB to reach Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb))
+		t.Run("Test Basic TLS Connectivity", tester.ConnectivitySucceeds(WithTls(mdb)))
+	})
+}