feat: retention policy

This commit makes the Barman cloud plugin support the enforcement of
retention policy as provided by the barman-cloud tool suite.

The first recoverability point and the last successful backup are
shown in the status of the ObjectStore resource for each involved
server name.

feat: implement recovery window

Signed-off-by: Leonardo Cecchi <[email protected]>

Author: leonardoce

api/v1/objectstore_types.go

@@ -27,20 +27,48 @@ type InstanceSidecarConfiguration struct {
 	// The environment to be explicitly passed to the sidecar
 	// +optional
 	Env []corev1.EnvVar `json:"env,omitempty"`
+
+	// The retentionCheckInterval defines the frequency at which the
+	// system checks and enforces retention policies.
+	// +kubebuilder:default:=1800
+	// +optional
+	RetentionPolicyIntervalSeconds int `json:"retentionPolicyIntervalSeconds,omitempty"`
 }
 
 // ObjectStoreSpec defines the desired state of ObjectStore.
 type ObjectStoreSpec struct {
+	// The configuration for the barman-cloud tool suite
 	Configuration barmanapi.BarmanObjectStoreConfiguration `json:"configuration"`
 
+	// RetentionPolicy is the retention policy to be used for backups
+	// and WALs (i.e. '60d'). The retention policy is expressed in the form
+	// of `XXu` where `XX` is a positive integer and `u` is in `[dwm]` -
+	// days, weeks, months.
+	// +kubebuilder:validation:Pattern=^[1-9][0-9]*[dwm]$
+	// +optional
+	RetentionPolicy string `json:"retentionPolicy,omitempty"`
+
 	// +optional
 	InstanceSidecarConfiguration InstanceSidecarConfiguration `json:"instanceSidecarConfiguration,omitempty"`
 }
 
 // ObjectStoreStatus defines the observed state of ObjectStore.
 type ObjectStoreStatus struct {
-	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
-	// Important: Run "make" to regenerate code after modifying this file
+	// ServerRecoveryWindow maps each server to its recovery window
+	ServerRecoveryWindow map[string]RecoveryWindow `json:"serverRecoveryWindow,omitempty"`
+}
+
+// RecoveryWindow represents the time span between the first
+// recoverability point and the last successful backup of a PostgreSQL
+// server, defining the period during which data can be restored.
+type RecoveryWindow struct {
+	// The first recoverability point in a PostgreSQL server refers to
+	// the earliest point in time to which the database can be
+	// restored.
+	FirstRecoverabilityPoint *metav1.Time `json:"firstRecoverabilityPoint,omitempty"`
+
+	// The last successful backup time
+	LastSuccessfulBackupTime *metav1.Time `json:"lastSuccussfulBackupTime,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/v1/zz_generated.deepcopy.go

@@ -40,9 +40,7 @@ spec:
             description: ObjectStoreSpec defines the desired state of ObjectStore.
             properties:
               configuration:
-                description: |-
-                  BarmanObjectStoreConfiguration contains the backup configuration
-                  using Barman against an S3-compatible object storage
+                description: The configuration for the barman-cloud tool suite
                 properties:
                   azureCredentials:
                     description: The credentials to use to upload data to Azure Blob
@@ -502,12 +500,49 @@ spec:
                       - name
                       type: object
                     type: array
+                  retentionPolicyIntervalSeconds:
+                    default: 1800
+                    description: |-
+                      The retentionCheckInterval defines the frequency at which the
+                      system checks and enforces retention policies.
+                    type: integer
                 type: object
+              retentionPolicy:
+                description: |-
+                  RetentionPolicy is the retention policy to be used for backups
+                  and WALs (i.e. '60d'). The retention policy is expressed in the form
+                  of `XXu` where `XX` is a positive integer and `u` is in `[dwm]` -
+                  days, weeks, months.
+                pattern: ^[1-9][0-9]*[dwm]$
+                type: string
             required:
             - configuration
             type: object
           status:
             description: ObjectStoreStatus defines the observed state of ObjectStore.
+            properties:
+              serverRecoveryWindow:
+                additionalProperties:
+                  description: |-
+                    RecoveryWindow represents the time span between the first
+                    recoverability point and the last successful backup of a PostgreSQL
+                    server, defining the period during which data can be restored.
+                  properties:
+                    firstRecoverabilityPoint:
+                      description: |-
+                        The first recoverability point in a PostgreSQL server refers to
+                        the earliest point in time to which the database can be
+                        restored.
+                      format: date-time
+                      type: string
+                    lastSuccussfulBackupTime:
+                      description: The last successful backup time
+                      format: date-time
+                      type: string
+                  type: object
+                description: ServerRecoveryWindow maps each server to its recovery
+                  window
+                type: object
             type: object
         required:
         - metadata

config/crd/bases/barmancloud.cnpg.io_objectstores.yaml

@@ -0,0 +1,29 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: cluster-example
+spec:
+  instances: 3
+  imagePullPolicy: Always
+
+  backup:
+    barmanObjectStore:
+      destinationPath: s3://backups/
+      endpointURL: http://minio:9000
+      s3Credentials:
+        accessKeyId:
+          name: minio
+          key: ACCESS_KEY_ID
+        secretAccessKey:
+          name: minio
+          key: ACCESS_SECRET_KEY
+      wal:
+        compression: gzip
+      data:
+        additionalCommandArgs:
+          - "--min-chunk-size=5MB"
+          - "--read-timeout=60"
+          - "-vv"
+
+  storage:
+    size: 1Gi

docs/examples/cluster-example-legacy.yaml

@@ -3,6 +3,9 @@ kind: ObjectStore
 metadata:
   name: minio-store
 spec:
+  retentionPolicy: "1m"
+  instanceSidecarConfiguration:
+    retentionPolicyIntervalSeconds: 30
   configuration:
     destinationPath: s3://backups/
     endpointURL: http://minio:9000

docs/examples/minio-store.yaml

@@ -0,0 +1,14 @@
+#!/usr/bin/env sh
+
+# This script builds the images of the barman cloud plugin, to be used
+# to quickly test images in a development environment.
+#
+# After each run, the built images will have these names:
+#
+# - `plugin-barman-cloud:dev`
+# - `plugin-barman-cloud-sidecar:dev`
+
+set -eu
+
+docker build -t plugin-barman-cloud:dev --file containers/Dockerfile.plugin .
+docker build -t plugin-barman-cloud-sidecar:dev --file containers/Dockerfile.sidecar .

hack/build-dev-image.sh

@@ -18,6 +18,7 @@ func NewCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			requiredSettings := []string{
 				"namespace",
+				"cluster-name",
 				"pod-name",
 				"spool-directory",
 			}
@@ -33,6 +34,7 @@ func NewCmd() *cobra.Command {
 	}
 
 	_ = viper.BindEnv("namespace", "NAMESPACE")
+	_ = viper.BindEnv("cluster-name", "CLUSTER_NAME")
 	_ = viper.BindEnv("pod-name", "POD_NAME")
 	_ = viper.BindEnv("pgdata", "PGDATA")
 	_ = viper.BindEnv("spool-directory", "SPOOL_DIRECTORY")

internal/cmd/instance/main.go

@@ -151,6 +151,8 @@ func (b BackupServiceImplementation) Backup(
 			"version":     metadata.Data.Version,
 			"name":        metadata.Data.Name,
 			"displayName": metadata.Data.DisplayName,
+			"clusterUID":  string(configuration.Cluster.ObjectMeta.UID),
+			"pluginName":  metadata.PluginName,
 		},
 	}, nil
 }

internal/cnpgi/instance/backup.go

@@ -8,6 +8,7 @@ import (
 	"github.com/spf13/viper"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -30,7 +31,10 @@ func init() {
 func Start(ctx context.Context) error {
 	setupLog := log.FromContext(ctx)
 	setupLog.Info("Starting barman cloud instance plugin")
+
 	podName := viper.GetString("pod-name")
+	clusterName := viper.GetString("cluster-name")
+	namespace := viper.GetString("namespace")
 
 	controllerOptions := ctrl.Options{
 		Scheme: scheme,
@@ -40,6 +44,7 @@ func Start(ctx context.Context) error {
 					&corev1.Secret{},
 					&barmancloudv1.ObjectStore{},
 					&cnpgv1.Cluster{},
+					&cnpgv1.Backup{},
 				},
 			},
 		},
@@ -51,10 +56,11 @@ func Start(ctx context.Context) error {
 		return err
 	}
 
+	customCacheClient := extendedclient.NewExtendedClient(mgr.GetClient())
+
 	if err := mgr.Add(&CNPGI{
-		Client:       extendedclient.NewExtendedClient(mgr.GetClient()),
-		InstanceName: podName,
-		// TODO: improve
+		Client:         customCacheClient,
+		InstanceName:   podName,
 		PGDataPath:     viper.GetString("pgdata"),
 		PGWALPath:      path.Join(viper.GetString("pgdata"), "pg_wal"),
 		SpoolDirectory: viper.GetString("spool-directory"),
@@ -64,6 +70,19 @@ func Start(ctx context.Context) error {
 		return err
 	}
 
+	if err := mgr.Add(&RetentionPolicyRunnable{
+		Client:   customCacheClient,
+		Recorder: mgr.GetEventRecorderFor("policy-runnable"),
+		ClusterKey: types.NamespacedName{
+			Namespace: namespace,
+			Name:      clusterName,
+		},
+		PodName: podName,
+	}); err != nil {
+		setupLog.Error(err, "unable to policy enforcement runnable")
+		return err
+	}
+
 	if err := mgr.Start(ctx); err != nil {
 		return err
 	}