K8SPG-586: Add delete-backups finalizer

deploy/cr.yaml

@@ -5,6 +5,7 @@ metadata:
 #  finalizers:
 #  - percona.com/delete-pvc
 #  - percona.com/delete-ssl
+#  - percona.com/delete-backups
 spec:
   crVersion: 2.6.0
 #  metadata:

e2e-tests/tests/demand-backup/01-create-cluster.yaml

@@ -9,6 +9,7 @@ commands:
       source ../../functions
 
       get_cr "demand-backup" ${RANDOM} \
+        | yq '.metadata.finalizers=["percona.com/delete-backups"]' \
         | yq '.spec.backups.pgbackrest.global.repo1-retention-full="2"' \
         | yq '.spec.backups.pgbackrest.global.repo1-retention-full-type="count"' \
         | yq '.spec.backups.pgbackrest.global.repo3-retention-full="2"' \

e2e-tests/tests/demand-backup/25-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 30
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: 25-pg-backup-objects
+data:
+  data: "0"

e2e-tests/tests/demand-backup/25-delete-cluster-with-finalizer.yaml

@@ -0,0 +1,19 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      kubectl delete pg -n "${NAMESPACE}" demand-backup
+      sleep 15
+
+      data=1
+
+      res=$(kubectl -n "${NAMESPACE}" get pg-backup 2>&1 >/dev/null)
+
+      if [[ $res == *$(echo "No resources found in ${NAMESPACE} namespace.")* ]]; then
+          data=0
+      fi
+
+      kubectl create configmap -n "${NAMESPACE}" 25-pg-backup-objects --from-literal=data="${data}"

e2e-tests/tests/scheduled-backup/01-create-cluster.yaml

@@ -8,4 +8,6 @@ commands:
 
       source ../../functions
 
-      get_cr "scheduled-backup" ${RANDOM} | kubectl -n "${NAMESPACE}" apply -f -
+      get_cr "scheduled-backup" ${RANDOM} \
+      | yq '.metadata.finalizers=["percona.com/delete-backups"]' \
+      | kubectl -n "${NAMESPACE}" apply -f -

e2e-tests/tests/scheduled-backup/19-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 30
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: 19-pg-backup-objects
+data:
+  data: "0"

e2e-tests/tests/scheduled-backup/19-delete-cluster-with-finalizer.yaml

@@ -0,0 +1,19 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      kubectl delete pg -n "${NAMESPACE}" scheduled-backup
+      sleep 15
+
+      data=1
+
+      res=$(kubectl -n "${NAMESPACE}" get pg-backup 2>&1 >/dev/null)
+
+      if [[ $res == *$(echo "No resources found in ${NAMESPACE} namespace.")* ]]; then
+          data=0
+      fi
+
+      kubectl create configmap -n "${NAMESPACE}" 19-pg-backup-objects --from-literal=data="${data}"

percona/controller/pgcluster/controller.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/md5"
 	"fmt"
+	"io"
 	"reflect"
 	"strings"
 	"time"
@@ -29,6 +30,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
+	"github.com/percona/percona-postgresql-operator/internal/controller/runtime"
 	"github.com/percona/percona-postgresql-operator/internal/logging"
 	"github.com/percona/percona-postgresql-operator/internal/naming"
 	perconaController "github.com/percona/percona-postgresql-operator/percona/controller"
@@ -61,10 +63,22 @@ type PGClusterReconciler struct {
 	Watchers             *registry.Registry
 	ExternalChan         chan event.GenericEvent
 	StopExternalWatchers chan event.DeleteEvent
+	PodExec              func(
+		ctx context.Context, namespace, pod, container string,
+		stdin io.Reader, stdout, stderr io.Writer, command ...string,
+	) error
 }
 
 // SetupWithManager adds the PerconaPGCluster controller to the provided runtime manager
 func (r *PGClusterReconciler) SetupWithManager(mgr manager.Manager) error {
+	if r.PodExec == nil {
+		var err error
+		r.PodExec, err = runtime.NewPodExecutor(mgr.GetConfig())
+		if err != nil {
+			return err
+		}
+	}
+
 	if err := r.CrunchyController.Watch(source.Kind(mgr.GetCache(), &corev1.Secret{}, r.watchSecrets())); err != nil {
 		return errors.Wrap(err, "unable to watch secrets")
 	}
@@ -196,6 +210,10 @@ func (r *PGClusterReconciler) Reconcile(ctx context.Context, request reconcile.R
 	if cr.DeletionTimestamp != nil {
 		log.Info("Deleting PerconaPGCluster", "deletionTimestamp", cr.DeletionTimestamp)
 
+		if err := r.runFinalizers(ctx, cr); err != nil {
+			return reconcile.Result{RequeueAfter: 5 * time.Second}, errors.Wrap(err, "run finalizers")
+		}
+
 		// We're deleting PostgresCluster explicitly to let Crunchy controller run its finalizers and not mess with us.
 		if err := r.Client.Delete(ctx, postgresCluster); client.IgnoreNotFound(err) != nil {
 			return ctrl.Result{RequeueAfter: 5 * time.Second}, errors.Wrap(err, "delete postgres cluster")
@@ -206,10 +224,6 @@ func (r *PGClusterReconciler) Reconcile(ctx context.Context, request reconcile.R
 			return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
 		}
 
-		if err := r.runFinalizers(ctx, cr); err != nil {
-			return reconcile.Result{RequeueAfter: 5 * time.Second}, errors.Wrap(err, "run finalizers")
-		}
-
 		return reconcile.Result{}, nil
 	}
 

percona/controller/pgcluster/finalizer.go

@@ -1,7 +1,10 @@
 package pgcluster
 
 import (
+	"bytes"
 	"context"
+	"fmt"
+	"strings"
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -150,6 +153,61 @@ func (r *PGClusterReconciler) stopExternalWatchers(ctx context.Context, cr *v2.P
 	return nil
 }
 
+func (r *PGClusterReconciler) deleteBackups(ctx context.Context, cr *v2.PerconaPGCluster) error {
+	log := logging.FromContext(ctx)
+
+	podList := &corev1.PodList{}
+	err := r.Client.List(ctx, podList, &client.ListOptions{
+		Namespace: cr.Namespace,
+		LabelSelector: labels.SelectorFromSet(map[string]string{
+			naming.LabelPerconaInstance:  cr.Name,
+			naming.LabelPerconaComponent: "pg",
+		}),
+	})
+
+	if err != nil {
+		return err
+	}
+
+	if len(podList.Items) == 0 {
+		return errors.New("no pods found")
+	}
+
+	log.Info("Deleting backups from all the repos configured")
+
+	pod := podList.Items[0]
+
+	var stdout, stderr bytes.Buffer
+	pgBackrestCmd := "pgbackrest --stanza=db --log-level-console=info stop; " +
+		"pgbackrest --stanza=db --log-level-console=info --repo=%s stanza-delete --force"
+
+	for _, repo := range cr.Spec.Backups.PGBackRest.Repos {
+		cmd := []string{"bash", "-ceu", "--", fmt.Sprintf(pgBackrestCmd, strings.TrimPrefix(repo.Name, "repo"))}
+		if err := r.PodExec(ctx, cr.Namespace, pod.Name, "database", nil, &stdout, &stderr, cmd...); err != nil {
+			return errors.Wrapf(err, "delete backups, stderr: %s", stdout.String()+" "+stderr.String())
+		}
+		log.Info("Deleted backups from repo", "repo", repo.Name)
+	}
+
+	pbList := new(v2.PerconaPGBackupList)
+	err = r.Client.List(ctx, pbList, &client.ListOptions{
+		Namespace: cr.Namespace,
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed to list backup jobs")
+	}
+
+	log.Info("Deleting all PGBackup objects")
+	for _, pgBackup := range pbList.Items {
+		if err := r.Client.Delete(ctx, &pgBackup); err != nil {
+			return errors.Wrapf(err, "delete backup %s/%s", pgBackup.Name, pgBackup.Namespace)
+		}
+		log.Info("Deleted PGBackup", "name", pgBackup.Name)
+	}
+
+	return nil
+}
+
 func (r *PGClusterReconciler) runFinalizers(ctx context.Context, cr *v2.PerconaPGCluster) error {
 	if err := r.runFinalizer(ctx, cr, v2.FinalizerDeletePVC, r.deletePVCAndSecrets); err != nil {
 		return errors.Wrapf(err, "run finalizer %s", v2.FinalizerDeletePVC)
@@ -159,6 +217,10 @@ func (r *PGClusterReconciler) runFinalizers(ctx context.Context, cr *v2.PerconaP
 		return errors.Wrapf(err, "run finalizer %s", v2.FinalizerDeleteSSL)
 	}
 
+	if err := r.runFinalizer(ctx, cr, v2.FinalizerDeleteBackups, r.deleteBackups); err != nil {
+		return errors.Wrapf(err, "run finalizer %s", v2.FinalizerDeleteBackups)
+	}
+
 	if err := r.runFinalizer(ctx, cr, v2.FinalizerStopWatchers, r.stopExternalWatchers); err != nil {
 		return errors.Wrapf(err, "run finalizer %s", v2.FinalizerStopWatchers)
 	}

pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go

@@ -959,9 +959,10 @@ func GetDefaultVersionServiceEndpoint() string {
 }
 
 const (
-	FinalizerDeletePVC    = "percona.com/delete-pvc"
-	FinalizerDeleteSSL    = "percona.com/delete-ssl"
-	FinalizerStopWatchers = "percona.com/stop-watchers" //nolint:gosec
+	FinalizerDeletePVC     = "percona.com/delete-pvc"
+	FinalizerDeleteSSL     = "percona.com/delete-ssl"
+	FinalizerStopWatchers  = "percona.com/stop-watchers" //nolint:gosec
+	FinalizerDeleteBackups = "percona.com/delete-backups"
 )
 
 const (