feat: retention policy

This commit makes the Barman cloud plugin support the enforcement of
retention policy as provided by the barman-cloud tool suite

Signed-off-by: Leonardo Cecchi <[email protected]>

Author: leonardoce

api/v1/objectstore_types.go

@@ -27,12 +27,27 @@ type InstanceSidecarConfiguration struct {
 	// The environment to be explicitly passed to the sidecar
 	// +optional
 	Env []corev1.EnvVar `json:"env,omitempty"`
+
+	// The retentionCheckInterval defines the frequency at which the
+	// system checks and enforces retention policies.
+	// +kubebuilder:default:=1800
+	// +optional
+	RetentionPolicyIntervalSeconds int `json:"retentionPolicyIntervalSeconds,omitempty"`
 }
 
 // ObjectStoreSpec defines the desired state of ObjectStore.
 type ObjectStoreSpec struct {
+	// The configuration for the barman-cloud tool suite
 	Configuration barmanapi.BarmanObjectStoreConfiguration `json:"configuration"`
 
+	// RetentionPolicy is the retention policy to be used for backups
+	// and WALs (i.e. '60d'). The retention policy is expressed in the form
+	// of `XXu` where `XX` is a positive integer and `u` is in `[dwm]` -
+	// days, weeks, months.
+	// +kubebuilder:validation:Pattern=^[1-9][0-9]*[dwm]$
+	// +optional
+	RetentionPolicy string `json:"retentionPolicy,omitempty"`
+
 	// +optional
 	InstanceSidecarConfiguration InstanceSidecarConfiguration `json:"instanceSidecarConfiguration,omitempty"`
 }

config/crd/bases/barmancloud.cnpg.io_objectstores.yaml

@@ -40,9 +40,7 @@ spec:
             description: ObjectStoreSpec defines the desired state of ObjectStore.
             properties:
               configuration:
-                description: |-
-                  BarmanObjectStoreConfiguration contains the backup configuration
-                  using Barman against an S3-compatible object storage
+                description: The configuration for the barman-cloud tool suite
                 properties:
                   azureCredentials:
                     description: The credentials to use to upload data to Azure Blob
@@ -502,7 +500,21 @@ spec:
                       - name
                       type: object
                     type: array
+                  retentionPolicyIntervalSeconds:
+                    default: 1800
+                    description: |-
+                      The retentionCheckInterval defines the frequency at which the
+                      system checks and enforces retention policies.
+                    type: integer
                 type: object
+              retentionPolicy:
+                description: |-
+                  RetentionPolicy is the retention policy to be used for backups
+                  and WALs (i.e. '60d'). The retention policy is expressed in the form
+                  of `XXu` where `XX` is a positive integer and `u` is in `[dwm]` -
+                  days, weeks, months.
+                pattern: ^[1-9][0-9]*[dwm]$
+                type: string
             required:
             - configuration
             type: object

docs/examples/cluster-example-legacy.yaml

@@ -0,0 +1,29 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: cluster-example
+spec:
+  instances: 3
+  imagePullPolicy: Always
+
+  backup:
+    barmanObjectStore:
+      destinationPath: s3://backups/
+      endpointURL: http://minio:9000
+      s3Credentials:
+        accessKeyId:
+          name: minio
+          key: ACCESS_KEY_ID
+        secretAccessKey:
+          name: minio
+          key: ACCESS_SECRET_KEY
+      wal:
+        compression: gzip
+      data:
+        additionalCommandArgs:
+          - "--min-chunk-size=5MB"
+          - "--read-timeout=60"
+          - "-vv"
+
+  storage:
+    size: 1Gi

docs/examples/minio-store.yaml

@@ -3,6 +3,9 @@ kind: ObjectStore
 metadata:
   name: minio-store
 spec:
+  retentionPolicy: "1m"
+  instanceSidecarConfiguration:
+    retentionPolicyIntervalSeconds: 30
   configuration:
     destinationPath: s3://backups/
     endpointURL: http://minio:9000

hack/build-dev-image.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env sh
+
+# This script builds the images of the barman cloud plugin and pushes
+# them in the same registry that would be created and used by a
+# standard CloudNative-PG development environment.
+#
+# Usage instructions:
+#
+#   cloudnative-pg$ ./hack/setup-cluster.sh -n 1 -r destroy create load deploy
+#   plugin-barman-cloud$ ./hack/build-dev-image.sh
+#
+# Having done that, you can use the images from your development
+# cluster:
+#
+# - `registry.dev:5000/plugin-barman-cloud:dev`
+# - `registry.dev:5000/plugin-barman-cloud-sidecar:dev`
+
+set -eu
+
+docker build -t 127.0.0.1:5000/plugin-barman-cloud:dev --file containers/Dockerfile.plugin .
+docker build -t 127.0.0.1:5000/plugin-barman-cloud-sidecar:dev --file containers/Dockerfile.sidecar .
+docker push 127.0.0.1:5000/plugin-barman-cloud:dev
+docker push 127.0.0.1:5000/plugin-barman-cloud-sidecar:dev

internal/cmd/instance/main.go

@@ -18,6 +18,7 @@ func NewCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			requiredSettings := []string{
 				"namespace",
+				"cluster-name",
 				"pod-name",
 				"spool-directory",
 			}
@@ -33,6 +34,7 @@ func NewCmd() *cobra.Command {
 	}
 
 	_ = viper.BindEnv("namespace", "NAMESPACE")
+	_ = viper.BindEnv("cluster-name", "CLUSTER_NAME")
 	_ = viper.BindEnv("pod-name", "POD_NAME")
 	_ = viper.BindEnv("pgdata", "PGDATA")
 	_ = viper.BindEnv("spool-directory", "SPOOL_DIRECTORY")

internal/cnpgi/instance/backup.go

@@ -151,6 +151,8 @@ func (b BackupServiceImplementation) Backup(
 			"version":     metadata.Data.Version,
 			"name":        metadata.Data.Name,
 			"displayName": metadata.Data.DisplayName,
+			"clusterUID":  string(configuration.Cluster.ObjectMeta.UID),
+			"pluginName":  metadata.PluginName,
 		},
 	}, nil
 }

internal/cnpgi/instance/manager.go

@@ -8,6 +8,7 @@ import (
 	"github.com/spf13/viper"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -30,7 +31,10 @@ func init() {
 func Start(ctx context.Context) error {
 	setupLog := log.FromContext(ctx)
 	setupLog.Info("Starting barman cloud instance plugin")
+
 	podName := viper.GetString("pod-name")
+	clusterName := viper.GetString("cluster-name")
+	namespace := viper.GetString("namespace")
 
 	controllerOptions := ctrl.Options{
 		Scheme: scheme,
@@ -40,6 +44,7 @@ func Start(ctx context.Context) error {
 					&corev1.Secret{},
 					&barmancloudv1.ObjectStore{},
 					&cnpgv1.Cluster{},
+					&cnpgv1.Backup{},
 				},
 			},
 		},
@@ -51,10 +56,11 @@ func Start(ctx context.Context) error {
 		return err
 	}
 
+	customCacheClient := extendedclient.NewExtendedClient(mgr.GetClient())
+
 	if err := mgr.Add(&CNPGI{
-		Client:       extendedclient.NewExtendedClient(mgr.GetClient()),
-		InstanceName: podName,
-		// TODO: improve
+		Client:         customCacheClient,
+		InstanceName:   podName,
 		PGDataPath:     viper.GetString("pgdata"),
 		PGWALPath:      path.Join(viper.GetString("pgdata"), "pg_wal"),
 		SpoolDirectory: viper.GetString("spool-directory"),
@@ -64,6 +70,19 @@ func Start(ctx context.Context) error {
 		return err
 	}
 
+	if err := mgr.Add(&RetentionPolicyRunnable{
+		Client:   customCacheClient,
+		Recorder: mgr.GetEventRecorderFor("policy-runnable"),
+		ClusterKey: types.NamespacedName{
+			Namespace: namespace,
+			Name:      clusterName,
+		},
+		PodName: podName,
+	}); err != nil {
+		setupLog.Error(err, "unable to policy enforcement runnable")
+		return err
+	}
+
 	if err := mgr.Start(ctx); err != nil {
 		return err
 	}