K8SPSMDB-1366: Fix reconfiguring PBM after sharding is toggled (#1907)

* K8SPSMDB-1366: Fix reconfiguring PBM after sharding is toggled

* fix init-deploy compare

---------

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>

Author: egegunes

e2e-tests/rs-shard-migration/compare/statefulset_some-name-rs0.yml

@@ -55,7 +55,6 @@ spec:
             - --encryptionKeyFile=/etc/mongodb-encryption/encryption-key
             - --wiredTigerCacheSizeGB=0.25
             - --wiredTigerIndexPrefixCompression=true
-            - --config=/etc/mongodb-config/mongod.conf
             - --quiet
           command:
             - /opt/percona/ps-entry.sh
@@ -132,8 +131,6 @@ spec:
             - mountPath: /etc/mongodb-ssl-internal
               name: ssl-internal
               readOnly: true
-            - mountPath: /etc/mongodb-config
-              name: config
             - mountPath: /opt/percona
               name: bin
             - mountPath: /etc/mongodb-encryption
@@ -142,6 +139,57 @@ spec:
             - mountPath: /etc/users-secret
               name: users-secret-file
           workingDir: /data/db
+        - args:
+            - pbm-agent-entrypoint
+          command:
+            - /opt/percona/pbm-entry.sh
+          env:
+            - name: PBM_AGENT_MONGODB_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  key: MONGODB_BACKUP_USER_ESCAPED
+                  name: internal-some-name-users
+                  optional: false
+            - name: PBM_AGENT_MONGODB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  key: MONGODB_BACKUP_PASSWORD_ESCAPED
+                  name: internal-some-name-users
+                  optional: false
+            - name: PBM_MONGODB_REPLSET
+              value: rs0
+            - name: PBM_MONGODB_PORT
+              value: "27017"
+            - name: PBM_AGENT_SIDECAR
+              value: "true"
+            - name: PBM_AGENT_SIDECAR_SLEEP
+              value: "5"
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
+            - name: PBM_MONGODB_URI
+              value: mongodb://$(PBM_AGENT_MONGODB_USERNAME):$(PBM_AGENT_MONGODB_PASSWORD)@localhost:$(PBM_MONGODB_PORT)/?tls=true&tlsCertificateKeyFile=/tmp/tls.pem&tlsCAFile=/etc/mongodb-ssl/ca.crt&tlsInsecure=true
+            - name: PBM_AGENT_TLS_ENABLED
+              value: "true"
+          imagePullPolicy: Always
+          name: backup-agent
+          resources: {}
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 1001
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          volumeMounts:
+            - mountPath: /etc/mongodb-ssl
+              name: ssl
+              readOnly: true
+            - mountPath: /opt/percona
+              name: bin
+              readOnly: true
+            - mountPath: /data/db
+              name: mongod-data
       dnsPolicy: ClusterFirst
       initContainers:
         - command:
@@ -177,11 +225,6 @@ spec:
             secretName: some-name-mongodb-keyfile
         - emptyDir: {}
           name: bin
-        - configMap:
-            defaultMode: 420
-            name: some-name-rs0-mongod
-            optional: true
-          name: config
         - name: some-name-mongodb-encryption-key
           secret:
             defaultMode: 288

e2e-tests/rs-shard-migration/conf/backup-minio.yml

@@ -0,0 +1,9 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDBBackup
+metadata:
+  finalizers:
+    - percona.com/delete-backup
+  name: backup-minio
+spec:
+  clusterName: some-name
+  storageName: minio

e2e-tests/rs-shard-migration/conf/some-name.yml

@@ -0,0 +1,40 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDB
+metadata:
+  name: some-name
+spec:
+  #platform: openshift
+  image:
+  imagePullPolicy: Always
+  allowUnsafeConfigurations: false
+  backup:
+    enabled: true
+    image: perconalab/percona-server-mongodb-operator:0.4.0-backup
+    storages:
+      minio:
+        type: s3
+        s3:
+          credentialsSecret: minio-secret
+          region: us-east-1
+          bucket: operator-testing
+          endpointUrl: http://minio-service:9000/
+          insecureSkipTLSVerify: false
+  replsets:
+  - name: rs0
+    affinity:
+      antiAffinityTopologyKey: none
+    resources:
+      limits:
+        cpu: 500m
+        memory: 0.5G
+      requests:
+        cpu: 100m
+        memory: 0.1G
+    volumeSpec:
+      persistentVolumeClaim:
+        resources:
+          requests:
+            storage: 1Gi
+    size: 3
+  secrets:
+    users: some-users

e2e-tests/rs-shard-migration/run

@@ -15,21 +15,28 @@ function get_shard_parameter() {
 }
 
 function main() {
+	cluster="some-name"
+	CLUSTER_SIZE=3
+
 	create_infra $namespace
 
 	desc 'create secrets and start client'
 	kubectl_bin apply -f $conf_dir/secrets.yml -f $conf_dir/client.yml
-	cluster="some-name"
-	CLUSTER_SIZE=3
 
-	spinup_psmdb ${cluster}-rs0 $conf_dir/${cluster}-rs0.yml
+	apply_s3_storage_secrets
+	deploy_minio
+
+	spinup_psmdb ${cluster}-rs0 ${test_dir}/conf/${cluster}.yml
 
 	desc 'check if statefulset created with expected config'
 	compare_kubectl "statefulset/${cluster}-rs0"
 
 	desc 'write data, read from all'
 	simple_data_check "${cluster}-rs0" ${CLUSTER_SIZE}
 
+	run_backup "minio" "backup-minio-1"
+	wait_backup "backup-minio-1"
+
 	desc 'initiate migration from replicaset to sharded cluster'
 	kubectl_bin patch psmdb/${cluster} --type json -p='[{"op":"add","path":"/spec/sharding","value":{"configsvrReplSet":{"size":'${CLUSTER_SIZE}',"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"size":1}}}]'
 	sleep 10
@@ -58,6 +65,9 @@ function main() {
 		exit 1
 	fi
 
+	run_backup "minio" "backup-minio-2"
+	wait_backup "backup-minio-2"
+
 	desc 'get back from sharded cluster to replicaset'
 	kubectl_bin patch psmdb/${cluster} --type json -p='[{"op":"remove","path":"/spec/sharding"}]'
 	sleep 20
@@ -74,6 +84,9 @@ function main() {
 		exit 1
 	fi
 
+	run_backup "minio" "backup-minio-3"
+	wait_backup "backup-minio-3"
+
 	desc 'cleanup CRDs and RBAC'
 	kubectl_bin delete -f "${src_dir}/deploy/crd.yaml" || :
 	kubectl_bin delete -f "${src_dir}/deploy/rbac.yaml" || :

pkg/controller/perconaservermongodb/psmdb_controller.go

@@ -580,9 +580,13 @@ func (r *ReconcilePerconaServerMongoDB) handleShardingToggle(ctx context.Context
 	getShardingStatus := func(cr *api.PerconaServerMongoDB) api.ConditionStatus {
 		if cr.Spec.Sharding.Enabled {
 			return api.ConditionTrue
-		} else {
-			return api.ConditionFalse
 		}
+
+		return api.ConditionFalse
+	}
+
+	toggleShardingStatus := func(s api.ConditionStatus) bool {
+		return s != api.ConditionTrue
 	}
 
 	condition := cr.Status.FindCondition(api.AppStateSharding)
@@ -598,7 +602,14 @@ func (r *ReconcilePerconaServerMongoDB) handleShardingToggle(ctx context.Context
 		return nil
 	}
 
-	cr.Spec.Sharding.Enabled = !cr.Spec.Sharding.Enabled
+	log := logf.FromContext(ctx)
+	if toggleShardingStatus(condition.Status) {
+		log.Info("Sharding is enabled, pausing the cluster")
+	} else {
+		log.Info("Sharding is disabled, pausing the cluster")
+	}
+
+	cr.Spec.Sharding.Enabled = toggleShardingStatus(condition.Status)
 	cr.Spec.Pause = true
 	if err := cr.CheckNSetDefaults(ctx, r.serverVersion.Platform); err != nil {
 		return errors.Wrap(err, "check and set defaults")
@@ -616,8 +627,10 @@ func (r *ReconcilePerconaServerMongoDB) handleShardingToggle(ctx context.Context
 		return nil
 	}
 
-	cr.Spec.Sharding.Enabled = !cr.Spec.Sharding.Enabled
+	cr.Spec.Sharding.Enabled = toggleShardingStatus(condition.Status)
 	cr.Spec.Pause = false
+	// toggling sharding status removes all PBM collections, so we need to reconfigure
+	cr.Status.BackupConfigHash = ""
 	if err := cr.CheckNSetDefaults(ctx, r.serverVersion.Platform); err != nil {
 		return errors.Wrap(err, "check and set defaults")
 	}