diff --git a/config/rbac/cluster/role.yaml b/config/rbac/cluster/role.yaml index 0b2a74cb8b..45467fc81c 100644 --- a/config/rbac/cluster/role.yaml +++ b/config/rbac/cluster/role.yaml @@ -101,14 +101,19 @@ rules: - delete - get - list + - patch - update - watch - apiGroups: - pgv2.percona.com resources: - perconapgbackups/finalizers - - perconapgclusters/finalizers + - perconapgclusters/status + - perconapgrestores/status + - perconapgupgrades/finalizers + - perconapgupgrades/status verbs: + - patch - update - apiGroups: - pgv2.percona.com @@ -133,12 +138,8 @@ rules: - apiGroups: - pgv2.percona.com resources: - - perconapgclusters/status - - perconapgrestores/status - - perconapgupgrades/finalizers - - perconapgupgrades/status + - perconapgclusters/finalizers verbs: - - patch - update - apiGroups: - pgv2.percona.com @@ -148,6 +149,7 @@ rules: - create - get - list + - patch - watch - apiGroups: - policy diff --git a/config/rbac/namespace/role.yaml b/config/rbac/namespace/role.yaml index 6cec1f747c..316361007e 100644 --- a/config/rbac/namespace/role.yaml +++ b/config/rbac/namespace/role.yaml @@ -101,14 +101,19 @@ rules: - delete - get - list + - patch - update - watch - apiGroups: - pgv2.percona.com resources: - perconapgbackups/finalizers - - perconapgclusters/finalizers + - perconapgclusters/status + - perconapgrestores/status + - perconapgupgrades/finalizers + - perconapgupgrades/status verbs: + - patch - update - apiGroups: - pgv2.percona.com @@ -133,12 +138,8 @@ rules: - apiGroups: - pgv2.percona.com resources: - - perconapgclusters/status - - perconapgrestores/status - - perconapgupgrades/finalizers - - perconapgupgrades/status + - perconapgclusters/finalizers verbs: - - patch - update - apiGroups: - pgv2.percona.com @@ -148,6 +149,7 @@ rules: - create - get - list + - patch - watch - apiGroups: - policy diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 08fc8d9375..1f06a7581d 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -45798,14 +45798,19 @@ rules: - delete - get - list + - patch - update - watch - apiGroups: - pgv2.percona.com resources: - perconapgbackups/finalizers - - perconapgclusters/finalizers + - perconapgclusters/status + - perconapgrestores/status + - perconapgupgrades/finalizers + - perconapgupgrades/status verbs: + - patch - update - apiGroups: - pgv2.percona.com @@ -45830,12 +45835,8 @@ rules: - apiGroups: - pgv2.percona.com resources: - - perconapgclusters/status - - perconapgrestores/status - - perconapgupgrades/finalizers - - perconapgupgrades/status + - perconapgclusters/finalizers verbs: - - patch - update - apiGroups: - pgv2.percona.com @@ -45845,6 +45846,7 @@ rules: - create - get - list + - patch - watch - apiGroups: - policy diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index cca501f56d..3464fda704 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -45798,14 +45798,19 @@ rules: - delete - get - list + - patch - update - watch - apiGroups: - pgv2.percona.com resources: - perconapgbackups/finalizers - - perconapgclusters/finalizers + - perconapgclusters/status + - perconapgrestores/status + - perconapgupgrades/finalizers + - perconapgupgrades/status verbs: + - patch - update - apiGroups: - pgv2.percona.com @@ -45830,12 +45835,8 @@ rules: - apiGroups: - pgv2.percona.com resources: - - perconapgclusters/status - - perconapgrestores/status - - perconapgupgrades/finalizers - - perconapgupgrades/status + - perconapgclusters/finalizers verbs: - - patch - update - apiGroups: - pgv2.percona.com @@ -45845,6 +45846,7 @@ rules: - create - get - list + - patch - watch - apiGroups: - policy diff --git a/deploy/cw-rbac.yaml b/deploy/cw-rbac.yaml index def6ca41b7..fb053043de 100644 --- a/deploy/cw-rbac.yaml +++ b/deploy/cw-rbac.yaml @@ -105,14 +105,19 @@ rules: - delete - get - list + - patch - update - watch - apiGroups: - pgv2.percona.com resources: - perconapgbackups/finalizers - - perconapgclusters/finalizers + - perconapgclusters/status + - perconapgrestores/status + - perconapgupgrades/finalizers + - perconapgupgrades/status verbs: + - patch - update - apiGroups: - pgv2.percona.com @@ -137,12 +142,8 @@ rules: - apiGroups: - pgv2.percona.com resources: - - perconapgclusters/status - - perconapgrestores/status - - perconapgupgrades/finalizers - - perconapgupgrades/status + - perconapgclusters/finalizers verbs: - - patch - update - apiGroups: - pgv2.percona.com @@ -152,6 +153,7 @@ rules: - create - get - list + - patch - watch - apiGroups: - policy diff --git a/deploy/rbac.yaml b/deploy/rbac.yaml index 01db434099..5d24b58335 100644 --- a/deploy/rbac.yaml +++ b/deploy/rbac.yaml @@ -105,14 +105,19 @@ rules: - delete - get - list + - patch - update - watch - apiGroups: - pgv2.percona.com resources: - perconapgbackups/finalizers - - perconapgclusters/finalizers + - perconapgclusters/status + - perconapgrestores/status + - perconapgupgrades/finalizers + - perconapgupgrades/status verbs: + - patch - update - apiGroups: - pgv2.percona.com @@ -137,12 +142,8 @@ rules: - apiGroups: - pgv2.percona.com resources: - - perconapgclusters/status - - perconapgrestores/status - - perconapgupgrades/finalizers - - perconapgupgrades/status + - perconapgclusters/finalizers verbs: - - patch - update - apiGroups: - pgv2.percona.com @@ -152,6 +153,7 @@ rules: - create - get - list + - patch - watch - apiGroups: - policy diff --git a/e2e-tests/functions b/e2e-tests/functions index 0c16637126..00788229a1 100644 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -97,6 +97,23 @@ deploy_operator_gh() { | kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f - } +remove_all_finalizers() { + resource_types=("pg-restore" "pg-backup" "pg") + + for resource in "${resource_types[@]}"; do + echo "removing all finalizers for $resource resources" + + kubectl -n "${NAMESPACE}" get "$resource" -o json | jq '.items[] | .metadata.name' -r | while IFS= read -r name; do + kubectl -n "${NAMESPACE}" delete "$resource" "$name" --wait=0 + + if [[ $(kubectl -n "${NAMESPACE}" get "$resource" "$name" -o yaml | yq '.metadata.finalizers | length') == "0" ]]; then + continue + fi + kubectl -n "${NAMESPACE}" patch "$resource" "$name" --type=json -p='[{"op": "remove", "path": "/metadata/finalizers"}]' + done + done +} + destroy_operator() { kubectl -n "${OPERATOR_NS:-$NAMESPACE}" delete deployment percona-postgresql-operator --force --grace-period=0 || true if [[ $OPERATOR_NS ]]; then @@ -765,7 +782,7 @@ get_postgresql_logs() { for pod in $(kubectl get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do local phase=$(kubectl -n ${NAMESPACE} get ${pod} -o jsonpath={".status.phase"}) - if [[ "${phase}" != "Running" ]]; then + if [[ ${phase} != "Running" ]]; then echo "Waiting for ${pod} to start running" continue fi diff --git a/e2e-tests/tests/custom-extensions/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/custom-extensions/99-remove-cluster-gracefully.yaml index 4394715613..ef975eec32 100644 --- a/e2e-tests/tests/custom-extensions/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/custom-extensions/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/custom-tls/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/custom-tls/99-remove-cluster-gracefully.yaml index 51e1c9aac2..dfffdbf4dd 100644 --- a/e2e-tests/tests/custom-tls/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/custom-tls/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/demand-backup/18-assert.yaml b/e2e-tests/tests/demand-backup/18-assert.yaml index d488dee912..4940c20280 100644 --- a/e2e-tests/tests/demand-backup/18-assert.yaml +++ b/e2e-tests/tests/demand-backup/18-assert.yaml @@ -7,8 +7,6 @@ kind: PostgresCluster metadata: name: demand-backup generation: 10 - annotations: - postgres-operator.crunchydata.com/pgbackrest-restore: demand-backup-restore ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster diff --git a/e2e-tests/tests/demand-backup/19-assert.yaml b/e2e-tests/tests/demand-backup/19-assert.yaml index 6f2eaf5ee3..a73b7ae0c7 100644 --- a/e2e-tests/tests/demand-backup/19-assert.yaml +++ b/e2e-tests/tests/demand-backup/19-assert.yaml @@ -7,8 +7,6 @@ kind: PostgresCluster metadata: name: demand-backup generation: 10 - annotations: - postgres-operator.crunchydata.com/pgbackrest-restore: demand-backup-restore ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster @@ -59,4 +57,4 @@ apiVersion: v1 metadata: name: 19-verify-restored-data data: - data: ' 100500' \ No newline at end of file + data: ' 100500' diff --git a/e2e-tests/tests/demand-backup/21-assert.yaml b/e2e-tests/tests/demand-backup/21-assert.yaml index 074b7e4b94..abb5dfe1ab 100644 --- a/e2e-tests/tests/demand-backup/21-assert.yaml +++ b/e2e-tests/tests/demand-backup/21-assert.yaml @@ -7,8 +7,6 @@ kind: PostgresCluster metadata: name: demand-backup generation: 12 - annotations: - postgres-operator.crunchydata.com/pgbackrest-restore: demand-backup-restore-azure ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster diff --git a/e2e-tests/tests/demand-backup/22-assert.yaml b/e2e-tests/tests/demand-backup/22-assert.yaml index 45e5778495..6219c4888a 100644 --- a/e2e-tests/tests/demand-backup/22-assert.yaml +++ b/e2e-tests/tests/demand-backup/22-assert.yaml @@ -7,8 +7,6 @@ kind: PostgresCluster metadata: name: demand-backup generation: 12 - annotations: - postgres-operator.crunchydata.com/pgbackrest-restore: demand-backup-restore-azure ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster @@ -59,4 +57,4 @@ apiVersion: v1 metadata: name: 22-verify-restored-data data: - data: ' 100500' \ No newline at end of file + data: ' 100500' diff --git a/e2e-tests/tests/demand-backup/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/demand-backup/99-remove-cluster-gracefully.yaml index bbdf4380c8..3da58d6371 100644 --- a/e2e-tests/tests/demand-backup/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/demand-backup/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/finalizers/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/finalizers/99-remove-cluster-gracefully.yaml index 2c090979ff..b8213eee7c 100644 --- a/e2e-tests/tests/finalizers/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/finalizers/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/init-deploy/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/init-deploy/99-remove-cluster-gracefully.yaml index 2c090979ff..b8213eee7c 100644 --- a/e2e-tests/tests/init-deploy/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/init-deploy/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/major-upgrade/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/major-upgrade/99-remove-cluster-gracefully.yaml index f9ab14f25c..dd7090425b 100644 --- a/e2e-tests/tests/major-upgrade/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/major-upgrade/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/migration-backup-s3/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/migration-backup-s3/99-remove-cluster-gracefully.yaml index adfba9aaaf..f8cc0acb5a 100644 --- a/e2e-tests/tests/migration-backup-s3/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/migration-backup-s3/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/migration-data-volumes/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/migration-data-volumes/99-remove-cluster-gracefully.yaml index adfba9aaaf..f8cc0acb5a 100644 --- a/e2e-tests/tests/migration-data-volumes/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/migration-data-volumes/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/monitoring/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/monitoring/99-remove-cluster-gracefully.yaml index 9116b7f89a..8b1f92b78e 100644 --- a/e2e-tests/tests/monitoring/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/monitoring/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/one-pod/07-assert.yaml b/e2e-tests/tests/one-pod/07-assert.yaml index 01410f5aa0..8dbaa92521 100644 --- a/e2e-tests/tests/one-pod/07-assert.yaml +++ b/e2e-tests/tests/one-pod/07-assert.yaml @@ -7,8 +7,6 @@ kind: PostgresCluster metadata: name: one-pod generation: 4 - annotations: - postgres-operator.crunchydata.com/pgbackrest-restore: one-pod-restore ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster diff --git a/e2e-tests/tests/one-pod/08-assert.yaml b/e2e-tests/tests/one-pod/08-assert.yaml index d05576ad96..1d6c7faf23 100644 --- a/e2e-tests/tests/one-pod/08-assert.yaml +++ b/e2e-tests/tests/one-pod/08-assert.yaml @@ -7,8 +7,6 @@ kind: PostgresCluster metadata: name: one-pod generation: 4 - annotations: - postgres-operator.crunchydata.com/pgbackrest-restore: one-pod-restore ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster @@ -53,4 +51,4 @@ apiVersion: v1 metadata: name: 08-verify-restored-data data: - data: ' 100500' \ No newline at end of file + data: ' 100500' diff --git a/e2e-tests/tests/one-pod/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/one-pod/99-remove-cluster-gracefully.yaml index af351153f5..74eca26884 100644 --- a/e2e-tests/tests/one-pod/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/one-pod/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/operator-self-healing/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/operator-self-healing/99-remove-cluster-gracefully.yaml index dc6861e14e..e448e36d0c 100644 --- a/e2e-tests/tests/operator-self-healing/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/operator-self-healing/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/pitr/05-assert.yaml b/e2e-tests/tests/pitr/05-assert.yaml index b51e3ba249..03546a2eaf 100644 --- a/e2e-tests/tests/pitr/05-assert.yaml +++ b/e2e-tests/tests/pitr/05-assert.yaml @@ -7,8 +7,6 @@ kind: PostgresCluster metadata: name: pitr generation: 4 - annotations: - postgres-operator.crunchydata.com/pgbackrest-restore: pitr-restore ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster diff --git a/e2e-tests/tests/pitr/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/pitr/99-remove-cluster-gracefully.yaml index 533a4408ea..2250c13713 100644 --- a/e2e-tests/tests/pitr/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/pitr/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/scaling/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/scaling/99-remove-cluster-gracefully.yaml index 4ab6144184..58b4cd3338 100644 --- a/e2e-tests/tests/scaling/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/scaling/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/scheduled-backup/10-assert.yaml b/e2e-tests/tests/scheduled-backup/10-assert.yaml index fd1960b025..5ecc9ff6da 100644 --- a/e2e-tests/tests/scheduled-backup/10-assert.yaml +++ b/e2e-tests/tests/scheduled-backup/10-assert.yaml @@ -7,8 +7,6 @@ kind: PostgresCluster metadata: name: scheduled-backup generation: 11 - annotations: - postgres-operator.crunchydata.com/pgbackrest-restore: s3-restore ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster diff --git a/e2e-tests/tests/scheduled-backup/13-assert.yaml b/e2e-tests/tests/scheduled-backup/13-assert.yaml index e6d49af169..d9d884990f 100644 --- a/e2e-tests/tests/scheduled-backup/13-assert.yaml +++ b/e2e-tests/tests/scheduled-backup/13-assert.yaml @@ -7,8 +7,6 @@ kind: PostgresCluster metadata: name: scheduled-backup generation: 13 - annotations: - postgres-operator.crunchydata.com/pgbackrest-restore: gcs-restore ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster diff --git a/e2e-tests/tests/scheduled-backup/16-assert.yaml b/e2e-tests/tests/scheduled-backup/16-assert.yaml index 37c59ce6a9..63acd7925d 100644 --- a/e2e-tests/tests/scheduled-backup/16-assert.yaml +++ b/e2e-tests/tests/scheduled-backup/16-assert.yaml @@ -7,8 +7,6 @@ kind: PostgresCluster metadata: name: scheduled-backup generation: 15 - annotations: - postgres-operator.crunchydata.com/pgbackrest-restore: azure-restore ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster diff --git a/e2e-tests/tests/scheduled-backup/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/scheduled-backup/99-remove-cluster-gracefully.yaml index 4ab6144184..58b4cd3338 100644 --- a/e2e-tests/tests/scheduled-backup/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/scheduled-backup/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/self-healing/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/self-healing/99-remove-cluster-gracefully.yaml index 5ec80ed3fb..742008261a 100644 --- a/e2e-tests/tests/self-healing/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/self-healing/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/sidecars/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/sidecars/99-remove-cluster-gracefully.yaml index 59f953f9dc..cbf47de570 100644 --- a/e2e-tests/tests/sidecars/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/sidecars/99-remove-cluster-gracefully.yaml @@ -16,6 +16,7 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/start-from-backup/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/start-from-backup/99-remove-cluster-gracefully.yaml index 583ec46c59..5e03516bd3 100644 --- a/e2e-tests/tests/start-from-backup/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/start-from-backup/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/tablespaces/08-assert.yaml b/e2e-tests/tests/tablespaces/08-assert.yaml index a6562dd3af..596cb7fd94 100644 --- a/e2e-tests/tests/tablespaces/08-assert.yaml +++ b/e2e-tests/tests/tablespaces/08-assert.yaml @@ -7,8 +7,6 @@ kind: PostgresCluster metadata: name: tablespaces generation: 4 - annotations: - postgres-operator.crunchydata.com/pgbackrest-restore: tablespaces-restore ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster diff --git a/e2e-tests/tests/tablespaces/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/tablespaces/99-remove-cluster-gracefully.yaml index 197c3a990b..2915b2e4a2 100644 --- a/e2e-tests/tests/tablespaces/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/tablespaces/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/telemetry-transfer/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/telemetry-transfer/99-remove-cluster-gracefully.yaml index 652c810866..c17aaa78d2 100644 --- a/e2e-tests/tests/telemetry-transfer/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/telemetry-transfer/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/upgrade-consistency/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/upgrade-consistency/99-remove-cluster-gracefully.yaml index f20d2298e7..7b881b6863 100644 --- a/e2e-tests/tests/upgrade-consistency/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/upgrade-consistency/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/upgrade-minor/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/upgrade-minor/99-remove-cluster-gracefully.yaml index a3051105d6..8f342bf561 100644 --- a/e2e-tests/tests/upgrade-minor/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/upgrade-minor/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/e2e-tests/tests/users/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/users/99-remove-cluster-gracefully.yaml index 6ee40cdc17..149c268d73 100644 --- a/e2e-tests/tests/users/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/users/99-remove-cluster-gracefully.yaml @@ -16,5 +16,6 @@ commands: source ../../functions + remove_all_finalizers destroy_operator timeout: 60 diff --git a/percona/controller/pgbackup/controller.go b/percona/controller/pgbackup/controller.go index 7a4178c59d..bfbeb0fb4e 100644 --- a/percona/controller/pgbackup/controller.go +++ b/percona/controller/pgbackup/controller.go @@ -3,15 +3,18 @@ package pgbackup import ( "context" "path" + "strings" "time" "github.com/pkg/errors" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -51,10 +54,10 @@ func (r *PGBackupReconciler) SetupWithManager(mgr manager.Manager) error { Complete(r)) } -// +kubebuilder:rbac:groups=pgv2.percona.com,resources=perconapgbackups,verbs=create;get;list;watch;update;delete +// +kubebuilder:rbac:groups=pgv2.percona.com,resources=perconapgbackups,verbs=create;get;list;watch;update;delete;patch // +kubebuilder:rbac:groups=pgv2.percona.com,resources=perconapgbackups/status,verbs=create;patch;update // +kubebuilder:rbac:groups=pgv2.percona.com,resources=perconapgclusters,verbs=get;list;create;update;patch;watch -// +kubebuilder:rbac:groups=pgv2.percona.com,resources=perconapgbackups/finalizers,verbs=update +// +kubebuilder:rbac:groups=pgv2.percona.com,resources=perconapgbackups/finalizers,verbs=update;patch // +kubebuilder:rbac:groups=postgres-operator.crunchydata.com,resources=postgresclusters,verbs=get;list;create;update;patch;watch // +kubebuilder:rbac:groups=postgres-operator.crunchydata.com,resources=postgresclusters/status,verbs=create;update;patch // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch @@ -75,11 +78,23 @@ func (r *PGBackupReconciler) Reconcile(ctx context.Context, request reconcile.Re pgBackup.Default() + if !pgBackup.DeletionTimestamp.IsZero() { + if _, err := runFinalizers(ctx, r.Client, pgBackup); err != nil { + return reconcile.Result{}, errors.Wrap(err, "failed to run finalizers") + } + return reconcile.Result{RequeueAfter: time.Second * 5}, nil + } + + if err := ensureFinalizers(ctx, r.Client, pgBackup); err != nil { + return reconcile.Result{}, errors.Wrap(err, "ensure finalizers") + } + pgCluster := &v2.PerconaPGCluster{} err := r.Client.Get(ctx, types.NamespacedName{Name: pgBackup.Spec.PGCluster, Namespace: request.Namespace}, pgCluster) if err != nil { return reconcile.Result{}, errors.Wrap(err, "get PostgresCluster") } + switch pgBackup.Status.State { case v2.BackupNew: if pgCluster.Spec.Pause != nil && *pgCluster.Spec.Pause { @@ -107,29 +122,37 @@ func (r *PGBackupReconciler) Reconcile(ctx context.Context, request reconcile.Re } } - pgBackup.Status.Destination = getDestination(pgCluster, pgBackup) - pgBackup.Status.Image = pgCluster.Spec.Backups.PGBackRest.Image - repo := getRepo(pgCluster, pgBackup) if repo == nil { return reconcile.Result{}, errors.Errorf("%s repo not defined", pgBackup.Spec.RepoName) } - pgBackup.Status.Repo = repo - pgBackup.Status.CRVersion = pgCluster.Spec.CRVersion - switch { - case repo.S3 != nil: - pgBackup.Status.StorageType = v2.PGBackupStorageTypeS3 - case repo.Azure != nil: - pgBackup.Status.StorageType = v2.PGBackupStorageTypeAzure - case repo.GCS != nil: - pgBackup.Status.StorageType = v2.PGBackupStorageTypeGCS - default: - pgBackup.Status.StorageType = v2.PGBackupStorageTypeFilesystem - } + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + bcp := new(v2.PerconaPGBackup) + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(pgBackup), bcp); err != nil { + return errors.Wrap(err, "get PGBackup") + } - pgBackup.Status.State = v2.BackupStarting - if err := r.Client.Status().Update(ctx, pgBackup); err != nil { + bcp.Status.Destination = getDestination(pgCluster, pgBackup) + bcp.Status.Image = pgCluster.Spec.Backups.PGBackRest.Image + bcp.Status.Repo = repo + bcp.Status.CRVersion = pgCluster.Spec.CRVersion + + switch { + case repo.S3 != nil: + bcp.Status.StorageType = v2.PGBackupStorageTypeS3 + case repo.Azure != nil: + bcp.Status.StorageType = v2.PGBackupStorageTypeAzure + case repo.GCS != nil: + bcp.Status.StorageType = v2.PGBackupStorageTypeGCS + default: + bcp.Status.StorageType = v2.PGBackupStorageTypeFilesystem + } + + bcp.Status.State = v2.BackupStarting + + return r.Client.Status().Update(ctx, bcp) + }); err != nil { return reconcile.Result{}, errors.Wrap(err, "update PGBackup status") } @@ -199,14 +222,13 @@ func (r *PGBackupReconciler) Reconcile(ctx context.Context, request reconcile.Re log.Info("Waiting for backup to complete") return reconcile.Result{RequeueAfter: time.Second * 5}, nil } - - rr, err := finishBackup(ctx, r.Client, pgBackup, job) + finished, err := runFinalizers(ctx, r.Client, pgBackup) if err != nil { - return reconcile.Result{}, err + return reconcile.Result{}, errors.Wrap(err, "failed to run finalizers") } - if rr != nil { + if !finished { log.Info("Waiting for crunchy reconciler to finish") - return *rr, nil + return reconcile.Result{RequeueAfter: time.Second * 5}, nil } if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { @@ -246,6 +268,71 @@ func (r *PGBackupReconciler) Reconcile(ctx context.Context, request reconcile.Re } } +func ensureFinalizers(ctx context.Context, cl client.Client, pgBackup *v2.PerconaPGBackup) error { + orig := pgBackup.DeepCopy() + + finalizers := []string{pNaming.FinalizerDeleteBackup} + finalizersChanged := false + for _, f := range finalizers { + if controllerutil.AddFinalizer(pgBackup, f) { + finalizersChanged = true + } + } + if !finalizersChanged { + return nil + } + + if err := cl.Patch(ctx, pgBackup.DeepCopy(), client.MergeFrom(orig)); err != nil { + return errors.Wrap(err, "remove finalizers") + } + return nil +} + +func runFinalizers(ctx context.Context, c client.Client, pgBackup *v2.PerconaPGBackup) (bool, error) { + pg := new(v2.PerconaPGCluster) + if err := c.Get(ctx, types.NamespacedName{Name: pgBackup.Spec.PGCluster, Namespace: pgBackup.Namespace}, pg); err != nil { + if !k8serrors.IsNotFound(err) { + return false, errors.Wrap(err, "get PostgresCluster") + } + + pg = nil + } + + finalizers := map[string]controller.FinalizerFunc[*v2.PerconaPGBackup]{ + pNaming.FinalizerDeleteBackup: func(ctx context.Context, pgBackup *v2.PerconaPGBackup) error { + if pg == nil { + return nil + } + job := new(batchv1.Job) + err := c.Get(ctx, types.NamespacedName{Name: pgBackup.Status.JobName, Namespace: pgBackup.Namespace}, job) + if client.IgnoreNotFound(err) != nil { + return errors.Wrap(err, "get backup job") + } + rr, err := finishBackup(ctx, c, pgBackup, job) + if err != nil { + return errors.Wrap(err, "failed to finish backup") + } + + if rr != nil && rr.RequeueAfter != 0 { + return controller.ErrKeepFinalizer + } + return nil + }, + } + + finished := true + for finalizer, f := range finalizers { + done, err := controller.RunFinalizer(ctx, c, pgBackup, finalizer, f) + if err != nil { + return false, errors.Wrapf(err, "run finalizer %s", finalizer) + } + if !done { + finished = false + } + } + return finished, nil +} + func getBackupInProgress(ctx context.Context, c client.Client, clusterName, ns string) (string, error) { pgCluster := &v2.PerconaPGCluster{} err := c.Get(ctx, types.NamespacedName{Name: clusterName, Namespace: ns}, pgCluster) @@ -372,6 +459,7 @@ func finishBackup(ctx context.Context, c client.Client, pgBackup *v2.PerconaPGBa if job.Labels[naming.LabelPGBackRestBackup] != string(naming.BackupManual) { return nil, nil } + runningBackup, err := getBackupInProgress(ctx, c, pgBackup.Spec.PGCluster, pgBackup.Namespace) if err != nil { return nil, errors.Wrap(err, "get backup in progress") @@ -424,12 +512,14 @@ func finishBackup(ctx context.Context, c client.Client, pgBackup *v2.PerconaPGBa return &reconcile.Result{RequeueAfter: time.Second * 5}, nil } - // Remove PGBackRest labels to prevent the job from being included in - // repoResources.manualBackupJobs and deleted by the cleanupRepoResources - // or reconcileManualBackup methods. + // Remove PGBackRest labels to prevent the job from being + // deleted by the cleanupRepoResources method. if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { j := new(batchv1.Job) if err := c.Get(ctx, client.ObjectKeyFromObject(job), j); err != nil { + if k8serrors.IsNotFound(err) { + return nil + } return errors.Wrap(err, "get job") } for k := range naming.PGBackRestLabels(pgBackup.Spec.PGCluster) { @@ -461,6 +551,30 @@ func finishBackup(ctx context.Context, c client.Client, pgBackup *v2.PerconaPGBa return &reconcile.Result{RequeueAfter: time.Second * 5}, nil } + if checkBackupJob(job) != v2.BackupSucceeded { + // Remove all crunchy labels to prevent the job from being included in + // repoResources.manualBackupJobs used in reconcileManualBackup method. + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + j := new(batchv1.Job) + if err := c.Get(ctx, client.ObjectKeyFromObject(job), j); err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return errors.Wrap(err, "get job") + } + + for k := range j.Labels { + if strings.HasPrefix(k, pNaming.PrefixCrunchy) { + delete(j.Labels, k) + } + } + + return c.Update(ctx, j) + }); err != nil { + return nil, errors.Wrap(err, "delete backup job labels") + } + } + return nil, nil } diff --git a/percona/controller/pgcluster/backup.go b/percona/controller/pgcluster/backup.go index 0392d492af..44639f1b1f 100644 --- a/percona/controller/pgcluster/backup.go +++ b/percona/controller/pgcluster/backup.go @@ -106,9 +106,6 @@ func (r *PGClusterReconciler) cleanupOutdatedBackups(ctx context.Context, cr *v2 } func (r *PGClusterReconciler) reconcileBackupJobs(ctx context.Context, cr *v2.PerconaPGCluster) error { - if err := checkBackupAnnotations(ctx, r.Client, cr); err != nil { - return errors.Wrap(err, "check backup annotation") - } for _, repo := range cr.Spec.Backups.PGBackRest.Repos { backupJobs, err := listBackupJobs(ctx, r.Client, cr, repo.Name) if err != nil { @@ -124,38 +121,6 @@ func (r *PGClusterReconciler) reconcileBackupJobs(ctx context.Context, cr *v2.Pe return nil } -func checkBackupAnnotations(ctx context.Context, cl client.Client, cr *v2.PerconaPGCluster) error { - backupName := cr.Annotations[pNaming.AnnotationBackupInProgress] - if backupName == "" { - return nil - } - - err := cl.Get(ctx, types.NamespacedName{Name: backupName, Namespace: cr.Namespace}, new(v2.PerconaPGBackup)) - if err == nil { - return nil - } - if client.IgnoreNotFound(err) != nil { - return errors.Wrapf(err, "failed to get backup %s", backupName) - } - - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - pgCluster := new(v2.PerconaPGCluster) - err := cl.Get(ctx, client.ObjectKeyFromObject(cr), pgCluster) - if err != nil { - return err - } - - delete(cr.Annotations, pNaming.AnnotationBackupInProgress) - delete(cr.Annotations, naming.PGBackRestBackup) - - return cl.Update(ctx, cr) - }) - if err != nil { - return errors.Wrap(err, "failed to update cluster annotation") - } - return nil -} - func reconcileBackupJob(ctx context.Context, cl client.Client, cr *v2.PerconaPGCluster, job batchv1.Job, repoName string) error { pb, err := findPGBackup(ctx, cl, cr, job, repoName) if err != nil { diff --git a/percona/controller/pgcluster/backup_test.go b/percona/controller/pgcluster/backup_test.go index c31f920ff2..a4abf6d55d 100644 --- a/percona/controller/pgcluster/backup_test.go +++ b/percona/controller/pgcluster/backup_test.go @@ -19,88 +19,6 @@ import ( "github.com/percona/percona-postgresql-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) -func TestCheckBackupAnnotations(t *testing.T) { - ctx := context.Background() - - const crName = "check-backup-annotations" - const backupName = "backup1" - const ns = crName - - cr, err := readDefaultCR(crName, ns) - if err != nil { - t.Fatal(err) - } - - pgBackup, err := readDefaultBackup(backupName, ns) - if err != nil { - t.Fatal(err) - } - pgBackup.Spec.PGCluster = crName - - fakeClient, err := buildFakeClient(ctx, cr, pgBackup) - if err != nil { - t.Fatalf("failed to build fake client: %v", err) - } - - reconciler := backupReconciler() - reconciler.Client = fakeClient - - reconcile := func() { - backupNsName := types.NamespacedName{Name: backupName, Namespace: ns} - - _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: backupNsName}) - if err != nil { - t.Fatal(err) - } - if err := fakeClient.Get(ctx, client.ObjectKeyFromObject(cr), cr); err != nil { - t.Fatal(err) - } - if err := fakeClient.Get(ctx, client.ObjectKeyFromObject(pgBackup), pgBackup); err != nil { - t.Fatal(err) - } - } - initialAnnotations := make(map[string]string) - for k, v := range cr.Annotations { - initialAnnotations[k] = v - } - - reconcile() - - startingAnnotations := make(map[string]string) - for k, v := range cr.Annotations { - startingAnnotations[k] = v - } - - if pgBackup.Status.State != v2.BackupStarting { - t.Fatal("backup state is not Starting") - } - - // Annotations should not be deleted if pgbackup is not deleted - if err := checkBackupAnnotations(ctx, fakeClient, cr); err != nil { - t.Fatal(err) - } - if err := fakeClient.Get(ctx, client.ObjectKeyFromObject(cr), cr); err != nil { - t.Fatal(err) - } - if !compareMaps(startingAnnotations, cr.Annotations) { - t.Fatal("cr annotations changed after check backup annotation") - } - - // Annotations should be deleted if pgbackup is deleted - if err := fakeClient.Delete(ctx, pgBackup); err != nil { - t.Fatal(err) - } - if err := checkBackupAnnotations(ctx, fakeClient, cr); err != nil { - t.Fatal(err) - } - if err := fakeClient.Get(ctx, client.ObjectKeyFromObject(cr), cr); err != nil { - t.Fatal(err) - } - if !compareMaps(cr.Annotations, initialAnnotations) { - t.Fatal("annotations are have different size", cr.Annotations, initialAnnotations) - } -} - func compareMaps(x map[string]string, y map[string]string) bool { if len(x) != len(y) { return false diff --git a/percona/controller/pgcluster/controller.go b/percona/controller/pgcluster/controller.go index bcbfe4143c..965764fefd 100644 --- a/percona/controller/pgcluster/controller.go +++ b/percona/controller/pgcluster/controller.go @@ -211,10 +211,6 @@ func (r *PGClusterReconciler) Reconcile(ctx context.Context, request reconcile.R }, } - if err := r.ensureFinalizers(ctx, cr); err != nil { - return reconcile.Result{}, errors.Wrap(err, "ensure finalizers") - } - if cr.DeletionTimestamp != nil { log.Info("Deleting PerconaPGCluster", "deletionTimestamp", cr.DeletionTimestamp) @@ -235,6 +231,10 @@ func (r *PGClusterReconciler) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, nil } + if err := r.ensureFinalizers(ctx, cr); err != nil { + return reconcile.Result{}, errors.Wrap(err, "ensure finalizers") + } + if err := r.reconcilePatroniVersionCheck(ctx, cr); err != nil { if errors.Is(err, errPatroniVersionCheckWait) { return reconcile.Result{ @@ -291,13 +291,17 @@ func (r *PGClusterReconciler) Reconcile(ctx context.Context, request reconcile.R } } - opRes, err := controllerutil.CreateOrUpdate(ctx, r.Client, postgresCluster, func() error { + var opRes controllerutil.OperationResult + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { var err error - postgresCluster, err = cr.ToCrunchy(ctx, postgresCluster, r.Client.Scheme()) + opRes, err = controllerutil.CreateOrUpdate(ctx, r.Client, postgresCluster, func() error { + var err error + postgresCluster, err = cr.ToCrunchy(ctx, postgresCluster, r.Client.Scheme()) + return err + }) return err - }) - if err != nil { + }); err != nil { return ctrl.Result{}, errors.Wrap(err, "update/create PostgresCluster") } @@ -311,8 +315,7 @@ func (r *PGClusterReconciler) Reconcile(ctx context.Context, request reconcile.R return ctrl.Result{}, errors.Wrap(err, "get PostgresCluster") } - err = r.updateStatus(ctx, cr, &postgresCluster.Status) - if err != nil { + if err := r.updateStatus(ctx, cr, &postgresCluster.Status); err != nil { return ctrl.Result{}, errors.Wrap(err, "update status") } @@ -851,14 +854,14 @@ func (r *PGClusterReconciler) stopExternalWatcher(ctx context.Context, cr *v2.Pe func (r *PGClusterReconciler) ensureFinalizers(ctx context.Context, cr *v2.PerconaPGCluster) error { for _, finalizer := range cr.Finalizers { - if finalizer == v2.FinalizerStopWatchers { + if finalizer == pNaming.FinalizerStopWatchers { return nil } } if *cr.Spec.Backups.TrackLatestRestorableTime { orig := cr.DeepCopy() - cr.Finalizers = append(cr.Finalizers, v2.FinalizerStopWatchers) + cr.Finalizers = append(cr.Finalizers, pNaming.FinalizerStopWatchers) if err := r.Client.Patch(ctx, cr.DeepCopy(), client.MergeFrom(orig)); err != nil { return errors.Wrap(err, "patch finalizers") } diff --git a/percona/controller/pgcluster/finalizer.go b/percona/controller/pgcluster/finalizer.go index 793d2e5899..8c90a1da9c 100644 --- a/percona/controller/pgcluster/finalizer.go +++ b/percona/controller/pgcluster/finalizer.go @@ -11,11 +11,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "github.com/percona/percona-postgresql-operator/internal/logging" "github.com/percona/percona-postgresql-operator/internal/naming" + "github.com/percona/percona-postgresql-operator/percona/controller" + pNaming "github.com/percona/percona-postgresql-operator/percona/naming" v2 "github.com/percona/percona-postgresql-operator/pkg/apis/pgv2.percona.com/v2" ) @@ -164,7 +165,6 @@ func (r *PGClusterReconciler) deleteBackups(ctx context.Context, cr *v2.PerconaP naming.LabelPerconaComponent: "pg", }), }) - if err != nil { return err } @@ -209,43 +209,16 @@ func (r *PGClusterReconciler) deleteBackups(ctx context.Context, cr *v2.PerconaP } func (r *PGClusterReconciler) runFinalizers(ctx context.Context, cr *v2.PerconaPGCluster) error { - if err := r.runFinalizer(ctx, cr, v2.FinalizerDeletePVC, r.deletePVCAndSecrets); err != nil { - return errors.Wrapf(err, "run finalizer %s", v2.FinalizerDeletePVC) - } - - if err := r.runFinalizer(ctx, cr, v2.FinalizerDeleteSSL, r.deleteTLSSecrets); err != nil { - return errors.Wrapf(err, "run finalizer %s", v2.FinalizerDeleteSSL) - } - - if err := r.runFinalizer(ctx, cr, v2.FinalizerDeleteBackups, r.deleteBackups); err != nil { - return errors.Wrapf(err, "run finalizer %s", v2.FinalizerDeleteBackups) - } - - if err := r.runFinalizer(ctx, cr, v2.FinalizerStopWatchers, r.stopExternalWatchers); err != nil { - return errors.Wrapf(err, "run finalizer %s", v2.FinalizerStopWatchers) - } - - return nil -} - -func (r *PGClusterReconciler) runFinalizer(ctx context.Context, cr *v2.PerconaPGCluster, finalizer string, f finalizerFunc) error { - if !controllerutil.ContainsFinalizer(cr, finalizer) { - return nil - } - - log := logging.FromContext(ctx) - log.Info("Running finalizer", "name", finalizer) - - orig := cr.DeepCopy() - - if err := f(ctx, cr); err != nil { - return errors.Wrapf(err, "run finalizer %s", finalizer) + finalizers := map[string]controller.FinalizerFunc[*v2.PerconaPGCluster]{ + pNaming.FinalizerDeletePVC: r.deletePVCAndSecrets, + pNaming.FinalizerDeleteSSL: r.deleteTLSSecrets, + pNaming.FinalizerStopWatchers: r.stopExternalWatchers, + pNaming.FinalizerDeleteBackups: r.deleteBackups, } - if controllerutil.RemoveFinalizer(cr, finalizer) { - log.Info("Removing finalizer", "name", finalizer) - if err := r.Client.Patch(ctx, cr, client.MergeFrom(orig)); err != nil { - return errors.Wrap(err, "remove finalizers") + for finalizer, f := range finalizers { + if _, err := controller.RunFinalizer(ctx, r.Client, cr, finalizer, f); err != nil { + return errors.Wrapf(err, "run finalizer %s", finalizer) } } diff --git a/percona/controller/pgcluster/finalizer_test.go b/percona/controller/pgcluster/finalizer_test.go index 7c4a578a5c..de9f3f218b 100644 --- a/percona/controller/pgcluster/finalizer_test.go +++ b/percona/controller/pgcluster/finalizer_test.go @@ -19,6 +19,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + pNaming "github.com/percona/percona-postgresql-operator/percona/naming" v2 "github.com/percona/percona-postgresql-operator/pkg/apis/pgv2.percona.com/v2" "github.com/percona/percona-postgresql-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -48,7 +49,7 @@ var _ = Describe("Finalizers", Ordered, func() { _ = k8sClient.Delete(ctx, namespace) }) - Context(v2.FinalizerDeletePVC, Ordered, func() { + Context(pNaming.FinalizerDeletePVC, Ordered, func() { crName := ns + "-with-delete-pvc" crNamespacedName := types.NamespacedName{Name: crName, Namespace: ns} When("with finalizer", func() { @@ -57,7 +58,7 @@ var _ = Describe("Finalizers", Ordered, func() { Expect(err).NotTo(HaveOccurred()) }) - controllerutil.AddFinalizer(cr, v2.FinalizerDeletePVC) + controllerutil.AddFinalizer(cr, pNaming.FinalizerDeletePVC) It("should create PerconaPGCluster", func() { status := cr.Status @@ -229,7 +230,7 @@ var _ = Describe("Finalizers", Ordered, func() { }) }) - Context(v2.FinalizerDeleteSSL, Ordered, func() { + Context(pNaming.FinalizerDeleteSSL, Ordered, func() { When("with finalizer", func() { crName := ns + "-with-delete-tls" crNamespacedName := types.NamespacedName{Name: crName, Namespace: ns} @@ -239,7 +240,7 @@ var _ = Describe("Finalizers", Ordered, func() { Expect(err).NotTo(HaveOccurred()) }) - controllerutil.AddFinalizer(cr, v2.FinalizerDeleteSSL) + controllerutil.AddFinalizer(cr, pNaming.FinalizerDeleteSSL) It("should create PerconaPGCluster", func() { status := cr.Status @@ -310,7 +311,7 @@ var _ = Describe("Finalizers", Ordered, func() { Expect(err).NotTo(HaveOccurred()) }) - controllerutil.RemoveFinalizer(cr, v2.FinalizerDeleteSSL) + controllerutil.RemoveFinalizer(cr, pNaming.FinalizerDeleteSSL) It("should create PerconaPGCluster", func() { status := cr.Status @@ -353,7 +354,7 @@ var _ = Describe("Finalizers", Ordered, func() { }) }) - Context(v2.FinalizerStopWatchers, Ordered, func() { + Context(pNaming.FinalizerStopWatchers, Ordered, func() { When("without finalizer", func() { crName := ns + "-without-stop-watchers" crNamespacedName := types.NamespacedName{Name: crName, Namespace: ns} @@ -363,7 +364,7 @@ var _ = Describe("Finalizers", Ordered, func() { Expect(err).NotTo(HaveOccurred()) }) - controllerutil.RemoveFinalizer(cr, v2.FinalizerStopWatchers) + controllerutil.RemoveFinalizer(cr, pNaming.FinalizerStopWatchers) It("should create PerconaPGCluster", func() { status := cr.Status @@ -386,7 +387,7 @@ var _ = Describe("Finalizers", Ordered, func() { return err == nil }, time.Second*15, time.Millisecond*250).Should(BeTrue()) - Expect(cr.Finalizers).Should(ContainElement(v2.FinalizerStopWatchers)) + Expect(cr.Finalizers).Should(ContainElement(pNaming.FinalizerStopWatchers)) }) }) }) diff --git a/percona/controller/pgcluster/schedule.go b/percona/controller/pgcluster/schedule.go index ad83229584..3c8d19997f 100644 --- a/percona/controller/pgcluster/schedule.go +++ b/percona/controller/pgcluster/schedule.go @@ -118,7 +118,7 @@ func (r *PGClusterReconciler) createScheduledBackup(log logr.Logger, backupName, }, } - if pb.CompareVersion("2.6.0") >= 0 && cr.Spec.Metadata != nil { + if cr.CompareVersion("2.6.0") >= 0 && cr.Spec.Metadata != nil { pb.Annotations = cr.Spec.Metadata.Annotations pb.Labels = cr.Spec.Metadata.Labels } diff --git a/percona/controller/pgrestore/controller.go b/percona/controller/pgrestore/controller.go index 08756e8aaa..db249f05ab 100644 --- a/percona/controller/pgrestore/controller.go +++ b/percona/controller/pgrestore/controller.go @@ -12,6 +12,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -41,7 +42,7 @@ func (r *PGRestoreReconciler) SetupWithManager(mgr manager.Manager) error { return builder.ControllerManagedBy(mgr).For(&v2.PerconaPGRestore{}).Complete(r) } -// +kubebuilder:rbac:groups=pgv2.percona.com,resources=perconapgrestores,verbs=get;list;watch;create +// +kubebuilder:rbac:groups=pgv2.percona.com,resources=perconapgrestores,verbs=get;list;watch;create;patch // +kubebuilder:rbac:groups=pgv2.percona.com,resources=perconapgrestores/status,verbs=patch;update // +kubebuilder:rbac:groups=pgv2.percona.com,resources=perconapgclusters,verbs=get;list;create;update;patch;watch // +kubebuilder:rbac:groups=postgres-operator.crunchydata.com,resources=postgresclusters,verbs=get;list;create;update;patch;watch @@ -61,6 +62,13 @@ func (r *PGRestoreReconciler) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, err } + if pgRestore.DeletionTimestamp != nil { + if err := runFinalizers(ctx, r.Client, pgRestore); err != nil { + return reconcile.Result{}, errors.Wrap(err, "failed to run finalizers") + } + return reconcile.Result{}, nil + } + if pgRestore.Status.State == v2.RestoreSucceeded || pgRestore.Status.State == v2.RestoreFailed { return reconcile.Result{}, nil } @@ -78,17 +86,17 @@ func (r *PGRestoreReconciler) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{RequeueAfter: time.Second * 5}, nil } - pgRestore.Status.State = v2.RestoreStarting - if err := r.Client.Status().Update(ctx, pgRestore); err != nil { - return reconcile.Result{}, errors.Wrap(err, "update PGRestore status") - } - if _, ok := pgRestore.Annotations[pNaming.AnnotationClusterBootstrapRestore]; !ok { if err := startRestore(ctx, r.Client, pgCluster, pgRestore); err != nil { return reconcile.Result{}, errors.Wrap(err, "start restore") } } + pgRestore.Status.State = v2.RestoreStarting + if err := r.Client.Status().Update(ctx, pgRestore); err != nil { + return reconcile.Result{}, errors.Wrap(err, "update PGRestore status") + } + return reconcile.Result{}, nil case v2.RestoreStarting: job := &batchv1.Job{} @@ -127,7 +135,7 @@ func (r *PGRestoreReconciler) Reconcile(ctx context.Context, request reconcile.R } if _, ok := pgRestore.Annotations[pNaming.AnnotationClusterBootstrapRestore]; !ok { - if err := disableRestore(ctx, r.Client, pgCluster); err != nil { + if err := disableRestore(ctx, r.Client, pgCluster, pgRestore); err != nil { return reconcile.Result{}, errors.Wrap(err, "disable restore") } } @@ -144,6 +152,53 @@ func (r *PGRestoreReconciler) Reconcile(ctx context.Context, request reconcile.R } } +func runFinalizers(ctx context.Context, c client.Client, pr *v2.PerconaPGRestore) error { + pg := new(v2.PerconaPGCluster) + if err := c.Get(ctx, types.NamespacedName{Name: pr.Spec.PGCluster, Namespace: pr.Namespace}, pg); err != nil { + if k8serrors.IsNotFound(err) { + pg = nil + } else { + return errors.Wrap(err, "get PostgresCluster") + } + } + + finalizers := map[string]controller.FinalizerFunc[*v2.PerconaPGRestore]{ + pNaming.FinalizerDeleteRestore: func(ctx context.Context, pr *v2.PerconaPGRestore) error { + if pg == nil { + return nil + } + return disableRestore(ctx, c, pg, pr) + }, + } + + for finalizer, f := range finalizers { + if _, err := controller.RunFinalizer(ctx, c, pr, finalizer, f); err != nil { + return errors.Wrapf(err, "run finalizer %s", finalizer) + } + } + return nil +} + +func ensureFinalizers(ctx context.Context, cl client.Client, pr *v2.PerconaPGRestore) error { + orig := pr.DeepCopy() + + finalizers := []string{pNaming.FinalizerDeleteRestore} + finalizersChanged := false + for _, f := range finalizers { + if controllerutil.AddFinalizer(pr, f) { + finalizersChanged = true + } + } + if !finalizersChanged { + return nil + } + + if err := cl.Patch(ctx, pr, client.MergeFrom(orig)); err != nil { + return errors.Wrap(err, "remove finalizers") + } + return nil +} + func startRestore(ctx context.Context, c client.Client, pg *v2.PerconaPGCluster, pr *v2.PerconaPGRestore) error { orig := pg.DeepCopy() @@ -152,6 +207,19 @@ func startRestore(ctx context.Context, c client.Client, pg *v2.PerconaPGCluster, } pg.Annotations[naming.PGBackRestRestore] = pr.Name + postgresCluster := new(v1beta1.PostgresCluster) + if err := c.Get(ctx, client.ObjectKeyFromObject(pg), postgresCluster); err != nil { + return errors.Wrap(err, "get PostgresCluster") + } + + origPostgres := postgresCluster.DeepCopy() + + postgresCluster.Status.PGBackRest.Restore = new(v1beta1.PGBackRestJobStatus) + + if err := c.Status().Patch(ctx, postgresCluster, client.MergeFrom(origPostgres)); err != nil { + return errors.Wrap(err, "patch PGCluster") + } + if pg.Spec.Backups.PGBackRest.Restore == nil { pg.Spec.Backups.PGBackRest.Restore = &v1beta1.PGBackRestRestore{ PostgresClusterDataSource: &v1beta1.PostgresClusterDataSource{}, @@ -167,10 +235,18 @@ func startRestore(ctx context.Context, c client.Client, pg *v2.PerconaPGCluster, return errors.Wrap(err, "patch PGCluster") } + if err := ensureFinalizers(ctx, c, pr); err != nil { + return errors.Wrap(err, "ensure restore finalizers") + } + return nil } -func disableRestore(ctx context.Context, c client.Client, pg *v2.PerconaPGCluster) error { +func disableRestore(ctx context.Context, c client.Client, pg *v2.PerconaPGCluster, pr *v2.PerconaPGRestore) error { + if pr.Status.State == v2.RestoreSucceeded || pr.Status.State == v2.RestoreFailed { + return nil + } + orig := pg.DeepCopy() if pg.Spec.Backups.PGBackRest.Restore == nil { @@ -182,6 +258,8 @@ func disableRestore(ctx context.Context, c client.Client, pg *v2.PerconaPGCluste fvar := false pg.Spec.Backups.PGBackRest.Restore.Enabled = &fvar + delete(pg.Annotations, naming.LabelPGBackRestRestore) + if err := c.Patch(ctx, pg, client.MergeFrom(orig)); err != nil { return errors.Wrap(err, "patch PGCluster") } diff --git a/percona/controller/utils.go b/percona/controller/utils.go index 3759a07619..a84b8ff567 100644 --- a/percona/controller/utils.go +++ b/percona/controller/utils.go @@ -8,8 +8,10 @@ import ( corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/manager" + "github.com/percona/percona-postgresql-operator/internal/logging" "github.com/percona/percona-postgresql-operator/internal/naming" ) @@ -77,3 +79,37 @@ func GetReadyInstancePod(ctx context.Context, c client.Client, clusterName, name } return nil, errors.New("no running instance found") } + +type FinalizerFunc[T client.Object] func(context.Context, T) error + +var ErrKeepFinalizer = errors.New("should keep finalizer") + +func RunFinalizer[T client.Object](ctx context.Context, cl client.Client, obj T, finalizer string, f FinalizerFunc[T]) (bool, error) { + if !controllerutil.ContainsFinalizer(obj, finalizer) { + return true, nil + } + + log := logging.FromContext(ctx) + log.Info("Running finalizer", "name", finalizer) + + orig, ok := obj.DeepCopyObject().(client.Object) + if !ok { + return false, errors.Errorf("failed to convert runtime.Object to client.Object") + } + + if err := f(ctx, obj); err != nil { + if errors.Is(err, ErrKeepFinalizer) { + return false, nil + } + return false, errors.Wrapf(err, "run finalizer %s", finalizer) + } + + if controllerutil.RemoveFinalizer(obj, finalizer) { + log.Info("Removing finalizer", "name", finalizer) + if err := cl.Patch(ctx, obj, client.MergeFrom(orig)); err != nil { + return false, errors.Wrap(err, "remove finalizers") + } + } + + return true, nil +} diff --git a/percona/naming/annotations.go b/percona/naming/annotations.go index b9e223215a..d9af7a8cb4 100644 --- a/percona/naming/annotations.go +++ b/percona/naming/annotations.go @@ -1,21 +1,12 @@ package naming -import ( - "strings" -) - -const ( - AnnotationPrefix = "pgv2.percona.com/" - CrunchyAnnotationPrefix = "postgres-operator.crunchydata.com/" -) - const ( // AnnotationPGBackrestBackup is the annotation that is added to a PerconaPGCluster to initiate a manual // backup. The value of the annotation will be a unique identifier for a backup Job (e.g. a // timestamp), which will be stored in the PostgresCluster status to properly track completion // of the Job. Also used to annotate the backup Job itself as needed to identify the backup // ID associated with a specific manual backup Job. - AnnotationPGBackrestBackup = AnnotationPrefix + "pgbackrest-backup" + AnnotationPGBackrestBackup = PrefixPerconaPGV2 + "pgbackrest-backup" // AnnotationPGBackrestBackupJobName is the annotation that is added to a PerconaPGClusterBackup. // The value of the annotation will be a name of an existing backup job @@ -29,39 +20,23 @@ const ( // restore. The value of the annotation will be a unique identfier for a restore Job (e.g. a // timestamp), which will be stored in the PostgresCluster status to properly track completion // of the Job. - AnnotationPGBackRestRestore = AnnotationPrefix + "pgbackrest-restore" + AnnotationPGBackRestRestore = PrefixPerconaPGV2 + "pgbackrest-restore" // AnnotationPMMSecretHash is the annotation that is added to instance annotations to // rollout restart PG pods in case PMM credentials are rotated. - AnnotationPMMSecretHash = AnnotationPrefix + "pmm-secret-hash" + AnnotationPMMSecretHash = PrefixPerconaPGV2 + "pmm-secret-hash" // AnnotationMonitorUserSecretHash is the annotation that is added to instance annotations to // rollout restart PG pods in case monitor user password is changed. - AnnotationMonitorUserSecretHash = AnnotationPrefix + "monitor-user-secret-hash" + AnnotationMonitorUserSecretHash = PrefixPerconaPGV2 + "monitor-user-secret-hash" // AnnotationBackupInProgress is the annotation that is added to PerconaPGCluster to // indicate that backup is in progress. - AnnotationBackupInProgress = AnnotationPrefix + "backup-in-progress" + AnnotationBackupInProgress = PrefixPerconaPGV2 + "backup-in-progress" // AnnotationClusterBootstrapRestore is the annotation that is added to PerconaPGRestore to // indicate that it is a cluster bootstrap restore. - AnnotationClusterBootstrapRestore = AnnotationPrefix + "cluster-bootstrap-restore" + AnnotationClusterBootstrapRestore = PrefixPerconaPGV2 + "cluster-bootstrap-restore" - AnnotationPatroniVersion = AnnotationPrefix + "patroni-version" + AnnotationPatroniVersion = PrefixPerconaPGV2 + "patroni-version" ) - -func ToCrunchyAnnotation(annotation string) string { - return replacePrefix(annotation, AnnotationPrefix, CrunchyAnnotationPrefix) -} - -func ToPerconaAnnotation(annotation string) string { - return replacePrefix(annotation, CrunchyAnnotationPrefix, AnnotationPrefix) -} - -func replacePrefix(s, oldPrefix, newPrefix string) string { - s, found := strings.CutPrefix(s, oldPrefix) - if found { - return newPrefix + s - } - return s -} diff --git a/percona/naming/finalizers.go b/percona/naming/finalizers.go new file mode 100644 index 0000000000..07ff8d677b --- /dev/null +++ b/percona/naming/finalizers.go @@ -0,0 +1,15 @@ +package naming + +// PerconaPGCluster finalizers +const ( + FinalizerDeletePVC = PrefixPercona + "delete-pvc" + FinalizerDeleteSSL = PrefixPercona + "delete-ssl" + FinalizerStopWatchers = PrefixPercona + "stop-watchers" //nolint:gosec + FinalizerDeleteBackups = PrefixPercona + "delete-backups" +) + +// PerconaPGRestore finalizers +const ( + FinalizerDeleteRestore = PrefixPercona + "delete-restore" //nolint:gosec + FinalizerDeleteBackup = PrefixPercona + "delete-backup" //nolint:gosec +) diff --git a/percona/naming/prefix.go b/percona/naming/prefix.go new file mode 100644 index 0000000000..f22c3b0181 --- /dev/null +++ b/percona/naming/prefix.go @@ -0,0 +1,25 @@ +package naming + +import "strings" + +const ( + PrefixPercona = "percona.com/" + PrefixPerconaPGV2 = "pgv2.percona.com/" + PrefixCrunchy = "postgres-operator.crunchydata.com/" +) + +func ToCrunchyAnnotation(annotation string) string { + return replacePrefix(annotation, PrefixPerconaPGV2, PrefixCrunchy) +} + +func ToPerconaAnnotation(annotation string) string { + return replacePrefix(annotation, PrefixCrunchy, PrefixPerconaPGV2) +} + +func replacePrefix(s, oldPrefix, newPrefix string) string { + s, found := strings.CutPrefix(s, oldPrefix) + if found { + return newPrefix + s + } + return s +} diff --git a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go index 9793480283..8c4cac36e6 100644 --- a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go +++ b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go @@ -982,13 +982,6 @@ func GetDefaultVersionServiceEndpoint() string { return DefaultVersionServiceEndpoint } -const ( - FinalizerDeletePVC = "percona.com/delete-pvc" - FinalizerDeleteSSL = "percona.com/delete-ssl" - FinalizerStopWatchers = "percona.com/stop-watchers" //nolint:gosec - FinalizerDeleteBackups = "percona.com/delete-backups" -) - const ( UserMonitoring = "monitor" )