K8SPSMDB-980: fix cluster deletion (#1329)

* K8SPSMDB-980: fix cluster deletion

https://jira.percona.com/browse/K8SPSMDB-980

* fix

* add comments

* set defaults to `terminationGracePeriodSeconds`

* increase default `terminationGracePeriodSeconds`

* fix tests

* fix `upgrade-consistency-sharded`

* So pods can have non-empty `deletionTimestamp` and
`ContainersReady: True` condition at the same time.
We should not increase counter of ready mongos pods if there is a timestamp

Also, there is a possible scenario when we have only 2 pods with the
`ContainersReady: True` condition without `deletionTimestamp` and
3 specified in `mongos.Size`.
In that case, operator will set mongos status to ready.
We should compare `mongos.Size` to the amount of existing mongos pods

---------

Co-authored-by: Viacheslav Sarzhan <[email protected]>

Author: pooknull

e2e-tests/arbiter/compare/statefulset_arbiter-clusterip-rs0-arbiter-oc.yml

@@ -162,7 +162,7 @@ spec:
       securityContext: {}
       serviceAccount: default
       serviceAccountName: default
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 60
       volumes:
       - name: arbiter-clusterip-mongodb-keyfile
         secret:

e2e-tests/arbiter/compare/statefulset_arbiter-clusterip-rs0-arbiter.yml

@@ -158,7 +158,7 @@ spec:
         fsGroup: 1001
       serviceAccount: default
       serviceAccountName: default
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 60
       volumes:
       - name: arbiter-clusterip-mongodb-keyfile
         secret:

e2e-tests/arbiter/compare/statefulset_arbiter-rs0-arbiter-oc.yml

@@ -161,7 +161,7 @@ spec:
       restartPolicy: Always
       schedulerName: default-scheduler
       securityContext: {}
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 60
       volumes:
         - name: arbiter-mongodb-keyfile
           secret:

e2e-tests/arbiter/compare/statefulset_arbiter-rs0-arbiter.yml

@@ -163,7 +163,7 @@ spec:
       schedulerName: default-scheduler
       securityContext:
         fsGroup: 1001
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 60
       volumes:
         - name: arbiter-mongodb-keyfile
           secret:

e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg-oc.yml

@@ -200,7 +200,7 @@ spec:
       securityContext: {}
       serviceAccount: default
       serviceAccountName: default
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 60
       volumes:
         - name: some-name-mongodb-keyfile
           secret:

e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml

@@ -203,7 +203,7 @@ spec:
         fsGroup: 1001
       serviceAccount: default
       serviceAccountName: default
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 60
       volumes:
         - name: some-name-mongodb-keyfile
           secret:

e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0-oc.yml

@@ -213,7 +213,7 @@ spec:
       securityContext: {}
       serviceAccount: default
       serviceAccountName: default
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 60
       volumes:
         - name: some-name-mongodb-keyfile
           secret:

e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml

@@ -216,7 +216,7 @@ spec:
         fsGroup: 1001
       serviceAccount: default
       serviceAccountName: default
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 60
       volumes:
         - name: some-name-mongodb-keyfile
           secret:

e2e-tests/default-cr/compare/statefulset_minimal-cluster-cfg-oc.yml

@@ -162,7 +162,7 @@ spec:
       securityContext: {}
       serviceAccount: default
       serviceAccountName: default
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 60
       volumes:
         - name: minimal-cluster-mongodb-keyfile
           secret:

e2e-tests/default-cr/compare/statefulset_minimal-cluster-cfg.yml

@@ -164,7 +164,7 @@ spec:
         fsGroup: 1001
       serviceAccount: default
       serviceAccountName: default
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 60
       volumes:
         - name: minimal-cluster-mongodb-keyfile
           secret: