WIP: CLOUDP-290847: Fix agent cert rotation

api/v1/mdb/mongodb_types.go

@@ -194,7 +194,7 @@ func (m *MongoDB) GetSecretsMountedIntoDBPod() []string {
 			secrets = append(secrets, tls)
 		}
 	}
-	agentCerts := m.GetSecurity().AgentClientCertificateSecretName(m.Name).Name
+	agentCerts := m.GetSecurity().AgentClientCertificateSecretName(m.Name)
 	if agentCerts != "" {
 		secrets = append(secrets, agentCerts)
 	}
@@ -851,7 +851,7 @@ func (s *Security) ShouldUseX509(currentAgentAuthMode string) bool {
 // AgentClientCertificateSecretName returns the name of the Secret that holds the agent
 // client TLS certificates.
 // If no custom name has been defined, it returns the default one.
-func (s Security) AgentClientCertificateSecretName(resourceName string) corev1.SecretKeySelector {
+func (s Security) AgentClientCertificateSecretName(resourceName string) string {
 	secretName := util.AgentSecretName
 
 	if s.CertificatesSecretsPrefix != "" {
@@ -861,10 +861,7 @@ func (s Security) AgentClientCertificateSecretName(resourceName string) corev1.S
 		secretName = s.Authentication.Agents.ClientCertificateSecretRefWrap.ClientCertificateSecretRef.Name
 	}
 
-	return corev1.SecretKeySelector{
-		Key:                  util.AutomationAgentPemSecretKey,
-		LocalObjectReference: corev1.LocalObjectReference{Name: secretName},
-	}
+	return secretName
 }
 
 // The customer has set ClientCertificateSecretRef. This signals that client certs are required,

api/v1/mdb/mongodb_types_test.go

@@ -382,15 +382,15 @@ func TestAgentClientCertificateSecretName(t *testing.T) {
 	rs := NewReplicaSetBuilder().SetSecurityTLSEnabled().EnableAuth([]AuthMode{util.X509}).Build()
 
 	// Default is the hardcoded "agent-certs"
-	assert.Equal(t, util.AgentSecretName, rs.GetSecurity().AgentClientCertificateSecretName(rs.Name).Name)
+	assert.Equal(t, util.AgentSecretName, rs.GetSecurity().AgentClientCertificateSecretName(rs.Name))
 
 	// If the top-level prefix is there, we use it
 	rs.Spec.Security.CertificatesSecretsPrefix = "prefix"
-	assert.Equal(t, fmt.Sprintf("prefix-%s-%s", rs.Name, util.AgentSecretName), rs.GetSecurity().AgentClientCertificateSecretName(rs.Name).Name)
+	assert.Equal(t, fmt.Sprintf("prefix-%s-%s", rs.Name, util.AgentSecretName), rs.GetSecurity().AgentClientCertificateSecretName(rs.Name))
 
 	// If the name is provided (deprecated) we return it
 	rs.GetSecurity().Authentication.Agents.ClientCertificateSecretRefWrap.ClientCertificateSecretRef.Name = "foo"
-	assert.Equal(t, "foo", rs.GetSecurity().AgentClientCertificateSecretName(rs.Name).Name)
+	assert.Equal(t, "foo", rs.GetSecurity().AgentClientCertificateSecretName(rs.Name))
 }
 
 func TestInternalClusterAuthSecretName(t *testing.T) {

controllers/om/automation_config_test.go

@@ -365,7 +365,7 @@ func TestCanResetAgentSSL(t *testing.T) {
 	ac.AgentSSL = &AgentSSL{
 		ClientCertificateMode: util.OptionalClientCertficates,
 		CAFilePath:            util.CAFilePathInContainer,
-		AutoPEMKeyFilePath:    util.AutomationAgentPemFilePath,
+		AutoPEMKeyFilePath:    "/fake/path/to/pem",
 	}
 
 	if err := ac.Apply(); err != nil {
@@ -374,7 +374,7 @@ func TestCanResetAgentSSL(t *testing.T) {
 
 	tls := cast.ToStringMap(ac.Deployment["tls"])
 	assert.Equal(t, tls["clientCertificateMode"], util.OptionalClientCertficates)
-	assert.Equal(t, tls["autoPEMKeyFilePath"], util.AutomationAgentPemFilePath)
+	assert.Equal(t, tls["autoPEMKeyFilePath"], "/fake/path/to/pem")
 	assert.Equal(t, tls["CAFilePath"], util.CAFilePathInContainer)
 
 	ac.AgentSSL = &AgentSSL{

controllers/om/backup_agent_config.go

@@ -49,8 +49,8 @@ func (bac *BackupAgentConfig) UnsetAgentPassword() {
 	bac.BackupAgentTemplate.Password = util.MergoDelete
 }
 
-func (bac *BackupAgentConfig) EnableX509Authentication(backupAgentSubject string) {
-	bac.BackupAgentTemplate.SSLPemKeyFile = util.AutomationAgentPemFilePath
+func (bac *BackupAgentConfig) EnableX509Authentication(backupAgentSubject, automationAgentPemFilePath string) {
+	bac.BackupAgentTemplate.SSLPemKeyFile = automationAgentPemFilePath
 	bac.SetAgentUserName(backupAgentSubject)
 }
 

controllers/om/backup_agent_test.go

@@ -32,7 +32,7 @@ func TestFieldsAreUpdatedBackupConfig(t *testing.T) {
 
 func TestBackupFieldsAreNotLost(t *testing.T) {
 	config := getTestBackupConfig()
-	config.EnableX509Authentication("namespace")
+	config.EnableX509Authentication("namespace", "/fake/path/to/pem")
 
 	assert.Contains(t, config.BackingMap, "logPath")
 	assert.Contains(t, config.BackingMap, "logRotate")
@@ -48,7 +48,7 @@ func TestBackupFieldsAreNotLost(t *testing.T) {
 func TestNestedFieldsAreNotLost(t *testing.T) {
 	config := getTestBackupConfig()
 
-	config.EnableX509Authentication("namespace")
+	config.EnableX509Authentication("namespace", "/fake/path/to/pem")
 
 	_ = config.Apply()
 

controllers/om/deployment/testing_utils.go

@@ -32,7 +32,7 @@ func CreateFromReplicaSet(mongoDBImage string, forceEnterprise bool, rs *mdb.Mon
 	}
 
 	d.MergeReplicaSet(
-		replicaset.BuildFromStatefulSet(mongoDBImage, forceEnterprise, sts, rs.GetSpec(), rs.Status.FeatureCompatibilityVersion),
+		replicaset.BuildFromStatefulSet(mongoDBImage, forceEnterprise, sts, rs.GetSpec(), rs.Status.FeatureCompatibilityVersion, ""),
 		rs.Spec.AdditionalMongodConfig.ToMap(),
 		lastConfig.ToMap(),
 		zap.S(),

controllers/om/monitoring_agent_config.go

@@ -45,9 +45,9 @@ func (m *MonitoringAgentConfig) UnsetAgentPassword() {
 	m.MonitoringAgentTemplate.Password = util.MergoDelete
 }
 
-func (m *MonitoringAgentConfig) EnableX509Authentication(MonitoringAgentSubject string) {
-	m.MonitoringAgentTemplate.SSLPemKeyFile = util.AutomationAgentPemFilePath
-	m.SetAgentUserName(MonitoringAgentSubject)
+func (m *MonitoringAgentConfig) EnableX509Authentication(monitoringAgentSubject, automationAgentPemFilePath string) {
+	m.MonitoringAgentTemplate.SSLPemKeyFile = automationAgentPemFilePath
+	m.SetAgentUserName(monitoringAgentSubject)
 }
 
 func (m *MonitoringAgentConfig) DisableX509Authentication() {

controllers/om/process/om_process.go

@@ -8,22 +8,15 @@ import (
 	mdbv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdb"
 	mdbmultiv1 "github.com/mongodb/mongodb-kubernetes/api/v1/mdbmulti"
 	"github.com/mongodb/mongodb-kubernetes/controllers/om"
-	"github.com/mongodb/mongodb-kubernetes/controllers/operator/certs"
 	"github.com/mongodb/mongodb-kubernetes/pkg/dns"
-	"github.com/mongodb/mongodb-kubernetes/pkg/util"
 )
 
-func CreateMongodProcessesWithLimit(mongoDBImage string, forceEnterprise bool, set appsv1.StatefulSet, dbSpec mdbv1.DbSpec, limit int, fcv string) []om.Process {
+func CreateMongodProcessesWithLimit(mongoDBImage string, forceEnterprise bool, set appsv1.StatefulSet, dbSpec mdbv1.DbSpec, limit int, fcv string, tlsCertPath string) []om.Process {
 	hostnames, names := dns.GetDnsForStatefulSetReplicasSpecified(set, dbSpec.GetClusterDomain(), limit, dbSpec.GetExternalDomain())
 	processes := make([]om.Process, len(hostnames))
 
-	certificateFileName := ""
-	if certificateHash, ok := set.Annotations[certs.CertHashAnnotationKey]; ok {
-		certificateFileName = fmt.Sprintf("%s/%s", util.TLSCertMountPath, certificateHash)
-	}
-
 	for idx, hostname := range hostnames {
-		processes[idx] = om.NewMongodProcess(names[idx], hostname, mongoDBImage, forceEnterprise, dbSpec.GetAdditionalMongodConfig(), dbSpec, certificateFileName, set.Annotations, fcv)
+		processes[idx] = om.NewMongodProcess(names[idx], hostname, mongoDBImage, forceEnterprise, dbSpec.GetAdditionalMongodConfig(), dbSpec, tlsCertPath, set.Annotations, fcv)
 	}
 
 	return processes

controllers/om/replicaset/om_replicaset.go

@@ -15,15 +15,15 @@ import (
 
 // BuildFromStatefulSet returns a replica set that can be set in the Automation Config
 // based on the given StatefulSet and MongoDB resource.
-func BuildFromStatefulSet(mongoDBImage string, forceEnterprise bool, set appsv1.StatefulSet, dbSpec mdbv1.DbSpec, fcv string) om.ReplicaSetWithProcesses {
-	return BuildFromStatefulSetWithReplicas(mongoDBImage, forceEnterprise, set, dbSpec, int(*set.Spec.Replicas), fcv)
+func BuildFromStatefulSet(mongoDBImage string, forceEnterprise bool, set appsv1.StatefulSet, dbSpec mdbv1.DbSpec, fcv string, tlsCertPath string) om.ReplicaSetWithProcesses {
+	return BuildFromStatefulSetWithReplicas(mongoDBImage, forceEnterprise, set, dbSpec, int(*set.Spec.Replicas), fcv, tlsCertPath)
 }
 
 // BuildFromStatefulSetWithReplicas returns a replica set that can be set in the Automation Config
 // based on the given StatefulSet and MongoDB spec. The amount of members is set by the replicas
 // parameter.
-func BuildFromStatefulSetWithReplicas(mongoDBImage string, forceEnterprise bool, set appsv1.StatefulSet, dbSpec mdbv1.DbSpec, replicas int, fcv string) om.ReplicaSetWithProcesses {
-	members := process.CreateMongodProcessesWithLimit(mongoDBImage, forceEnterprise, set, dbSpec, replicas, fcv)
+func BuildFromStatefulSetWithReplicas(mongoDBImage string, forceEnterprise bool, set appsv1.StatefulSet, dbSpec mdbv1.DbSpec, replicas int, fcv string, tlsCertPath string) om.ReplicaSetWithProcesses {
+	members := process.CreateMongodProcessesWithLimit(mongoDBImage, forceEnterprise, set, dbSpec, replicas, fcv, tlsCertPath)
 	replicaSet := om.NewReplicaSet(set.Name, dbSpec.GetMongoDBVersion())
 	rsWithProcesses := om.NewReplicaSetWithProcesses(replicaSet, members, dbSpec.GetMemberOptions())
 	rsWithProcesses.SetHorizons(dbSpec.GetHorizonConfig())

controllers/operator/appdbreplicaset_controller.go

@@ -1660,7 +1660,8 @@ func (r *ReconcileAppDbReplicaSet) tryConfigureMonitoringInOpsManager(ctx contex
 		Mechanisms:         []string{util.SCRAM},
 		ClientCertificates: util.OptionalClientCertficates,
 		AutoUser:           util.AutomationAgentUserName,
-		CAFilePath:         util.CAFilePathInContainer,
+		// TODO: add AutoPEMKeyFilePath
+		CAFilePath: util.CAFilePathInContainer,
 	}
 	err = authentication.Configure(conn, opts, false, log)
 	if err != nil {

controllers/operator/authentication/authentication.go

@@ -43,6 +43,8 @@ type Options struct {
 	// so it is possible to use other auth mechanisms without needing to provide client certs.
 	ClientCertificates string
 
+	AutoPEMKeyFilePath string
+
 	CAFilePath string
 
 	// Use Agent Client Auth
@@ -348,7 +350,7 @@ func addOrRemoveAgentClientCertificate(conn om.Connection, opts Options, log *za
 
 		if opts.AgentsShouldUseClientAuthentication {
 			ac.AgentSSL = &om.AgentSSL{
-				AutoPEMKeyFilePath:    util.AutomationAgentPemFilePath,
+				AutoPEMKeyFilePath:    opts.AutoPEMKeyFilePath,
 				CAFilePath:            opts.CAFilePath,
 				ClientCertificateMode: opts.ClientCertificates,
 			}

controllers/operator/authentication/x509.go

@@ -29,7 +29,7 @@ func (x *connectionX509) EnableAgentAuthentication(conn om.Connection, opts Opti
 		auth.KeyFile = util.AutomationAgentKeyFilePathInContainer
 		auth.KeyFileWindows = util.AutomationAgentWindowsKeyFilePath
 		ac.AgentSSL = &om.AgentSSL{
-			AutoPEMKeyFilePath:    util.AutomationAgentPemFilePath,
+			AutoPEMKeyFilePath:    opts.AutoPEMKeyFilePath,
 			CAFilePath:            opts.CAFilePath,
 			ClientCertificateMode: opts.ClientCertificates,
 		}
@@ -46,7 +46,7 @@ func (x *connectionX509) EnableAgentAuthentication(conn om.Connection, opts Opti
 
 	log.Info("Configuring backup agent user")
 	err = conn.ReadUpdateBackupAgentConfig(func(config *om.BackupAgentConfig) error {
-		config.EnableX509Authentication(opts.AutomationSubject)
+		config.EnableX509Authentication(opts.AutomationSubject, opts.AutoPEMKeyFilePath)
 		config.SetLdapGroupDN(opts.AutoLdapGroupDN)
 		return nil
 	}, log)
@@ -56,7 +56,7 @@ func (x *connectionX509) EnableAgentAuthentication(conn om.Connection, opts Opti
 
 	log.Info("Configuring monitoring agent user")
 	return conn.ReadUpdateMonitoringAgentConfig(func(config *om.MonitoringAgentConfig) error {
-		config.EnableX509Authentication(opts.AutomationSubject)
+		config.EnableX509Authentication(opts.AutomationSubject, opts.AutoPEMKeyFilePath)
 		config.SetLdapGroupDN(opts.AutoLdapGroupDN)
 		return nil
 	}, log)

controllers/operator/certs/certificates.go

@@ -31,7 +31,6 @@ type certDestination string
 
 const (
 	OperatorGeneratedCertSuffix = "-pem"
-	CertHashAnnotationKey       = "certHash"
 
 	Unused     = "unused"
 	Database   = "database"
@@ -310,14 +309,16 @@ func ValidateCertificates(ctx context.Context, secretGetter secret.Getter, name,
 
 // VerifyAndEnsureClientCertificatesForAgentsAndTLSType ensures that agent certs are present and correct, and returns whether they are of the kubernetes.io/tls type.
 // If the secret is of type kubernetes.io/tls, it creates a new secret containing the concatenation fo the tls.crt and tls.key fields
-func VerifyAndEnsureClientCertificatesForAgentsAndTLSType(ctx context.Context, secretReadClient, secretWriteClient secrets.SecretClient, secret types.NamespacedName) error {
+func VerifyAndEnsureClientCertificatesForAgentsAndTLSType(ctx context.Context, secretReadClient, secretWriteClient secrets.SecretClient, secret types.NamespacedName, log *zap.SugaredLogger) error {
 	needToCreatePEM := false
 	var secretData map[string][]byte
 	var s corev1.Secret
 	var err error
+	var databaseSecretPath string
 
 	if vault.IsVaultSecretBackend() {
 		needToCreatePEM = true
+		databaseSecretPath = secretReadClient.VaultClient.DatabaseSecretPath()
 		secretData, err = secretReadClient.VaultClient.ReadSecretBytes(fmt.Sprintf("%s/%s/%s", secretReadClient.VaultClient.DatabaseSecretPath(), secret.Namespace, secret.Name))
 		if err != nil {
 			return err
@@ -339,11 +340,8 @@ func VerifyAndEnsureClientCertificatesForAgentsAndTLSType(ctx context.Context, s
 			return err
 		}
 
-		dataMap := map[string]string{
-			util.AutomationAgentPemSecretKey: data,
-		}
-
-		return CreateOrUpdatePEMSecret(ctx, secretWriteClient, secret, dataMap, []metav1.OwnerReference{}, Database)
+		secretHash := enterprisepem.ReadHashFromSecret(ctx, secretReadClient, secret.Namespace, secret.Name, databaseSecretPath, log)
+		return CreateOrUpdatePEMSecretWithPreviousCert(ctx, secretWriteClient, secret, secretHash, data, []metav1.OwnerReference{}, Database)
 	}
 
 	return validatePemSecret(s, util.AutomationAgentPemSecretKey, nil)

controllers/operator/common_controller.go

@@ -30,6 +30,7 @@ import (
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/authentication"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/certs"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/construct"
+	enterprisepem "github.com/mongodb/mongodb-kubernetes/controllers/operator/pem"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/secrets"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/watch"
 	"github.com/mongodb/mongodb-kubernetes/controllers/operator/workflow"
@@ -231,7 +232,7 @@ func (r *ReconcileCommonController) SetupCommonWatchers(watcherResource WatcherR
 		} else {
 			secretNames = []string{security.MemberCertificateSecretName(resourceNameForSecret)}
 			if security.ShouldUseX509("") {
-				secretNames = append(secretNames, security.AgentClientCertificateSecretName(resourceNameForSecret).Name)
+				secretNames = append(secretNames, security.AgentClientCertificateSecretName(resourceNameForSecret))
 			}
 		}
 		r.resourceWatcher.RegisterWatchedTLSResources(objectToReconcile, security.TLSConfig.CA, secretNames)
@@ -504,6 +505,7 @@ func (r *ReconcileCommonController) updateOmAuthentication(ctx context.Context,
 				return workflow.Failed(xerrors.Errorf("error configuring agent subjects: %w", err)), false
 			}
 			authOpts.AgentsShouldUseClientAuthentication = ar.GetSecurity().ShouldUseClientCertificates()
+			authOpts.AutoPEMKeyFilePath = util.PvcMmsHomeMountPath + "/" + util.AgentSecretName + "/" + agentCertSecretSelector.Key
 		}
 		if ar.GetSecurity().ShouldUseLDAP(ac.Auth.AutoAuthMechanism) {
 			secretRef := ar.GetSecurity().Authentication.Agents.AutomationPasswordSecretRef
@@ -595,16 +597,22 @@ func (r *ReconcileCommonController) readAgentSubjectsFromSecret(ctx context.Cont
 }
 
 func (r *ReconcileCommonController) clearProjectAuthenticationSettings(ctx context.Context, conn om.Connection, mdb *mdbv1.MongoDB, processNames []string, log *zap.SugaredLogger) error {
-	secretKeySelector := mdb.Spec.Security.AgentClientCertificateSecretName(mdb.Name)
+	agentCertSecretName := mdb.Spec.Security.AgentClientCertificateSecretName(mdb.Name)
+
 	agentSecret := &corev1.Secret{}
-	if err := r.client.Get(ctx, kube.ObjectKey(mdb.Namespace, secretKeySelector.Name), agentSecret); client.IgnoreNotFound(err) != nil {
+	if err := r.client.Get(ctx, kube.ObjectKey(mdb.Namespace, agentCertSecretName), agentSecret); client.IgnoreNotFound(err) != nil {
 		return nil
 	}
 
 	if agentSecret.Type == corev1.SecretTypeTLS {
-		secretKeySelector.Name = fmt.Sprintf("%s%s", secretKeySelector.Name, certs.OperatorGeneratedCertSuffix)
+		agentCertSecretName = fmt.Sprintf("%s%s", agentCertSecretName, certs.OperatorGeneratedCertSuffix)
 	}
 
+	agentCertHash := enterprisepem.ReadHashFromSecret(ctx, r.SecretClient, mdb.Namespace, agentCertSecretName, "", log)
+	secretKeySelector := corev1.SecretKeySelector{
+		LocalObjectReference: corev1.LocalObjectReference{Name: agentCertSecretName},
+		Key:                  agentCertHash,
+	}
 	userOpts, err := r.readAgentSubjectsFromSecret(ctx, mdb.Namespace, secretKeySelector, log)
 	err = client.IgnoreNotFound(err)
 	if err != nil {
@@ -631,8 +639,8 @@ func (r *ReconcileCommonController) ensureX509SecretAndCheckTLSType(ctx context.
 		if !security.IsTLSEnabled() {
 			return workflow.Failed(xerrors.Errorf("Authentication mode for project is x509 but this MDB resource is not TLS enabled"))
 		}
-		agentSecretName := security.AgentClientCertificateSecretName(configurator.GetName()).Name
-		err := certs.VerifyAndEnsureClientCertificatesForAgentsAndTLSType(ctx, configurator.GetSecretReadClient(), configurator.GetSecretWriteClient(), kube.ObjectKey(configurator.GetNamespace(), agentSecretName))
+		agentSecretName := security.AgentClientCertificateSecretName(configurator.GetName())
+		err := certs.VerifyAndEnsureClientCertificatesForAgentsAndTLSType(ctx, configurator.GetSecretReadClient(), configurator.GetSecretWriteClient(), kube.ObjectKey(configurator.GetNamespace(), agentSecretName), log)
 		if err != nil {
 			return workflow.Failed(err)
 		}

controllers/operator/common_controller_test.go

@@ -418,7 +418,7 @@ func TestSecretWatcherWithAllResources(t *testing.T) {
 	// TODO: unify the watcher setup with the secret creation/mounting code in database creation
 	memberCert := rs.GetSecurity().MemberCertificateSecretName(rs.Name)
 	internalAuthCert := rs.GetSecurity().InternalClusterAuthSecretName(rs.Name)
-	agentCert := rs.GetSecurity().AgentClientCertificateSecretName(rs.Name).Name
+	agentCert := rs.GetSecurity().AgentClientCertificateSecretName(rs.Name)
 
 	expected := map[watch.Object][]types.NamespacedName{
 		{ResourceType: watch.ConfigMap, Resource: kube.ObjectKey(mock.TestNamespace, mock.TestProjectConfigMapName)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)},