feat: retention policy (#191)

This commit makes the Barman cloud plugin support the enforcement of
retention policy as provided by the barman-cloud tool suite.

The first recoverability point and the last successful backup are
shown in the status of the ObjectStore resource for each involved
server name.

Signed-off-by: Leonardo Cecchi <[email protected]>
Signed-off-by: Armando Ruocco <[email protected]>
Signed-off-by: Marco Nenciarini <[email protected]>
Co-authored-by: Armando Ruocco <[email protected]>
Co-authored-by: Marco Nenciarini <[email protected]>

Author: 3 people

api/v1/objectstore_types.go

@@ -27,20 +27,48 @@ type InstanceSidecarConfiguration struct {
 	// The environment to be explicitly passed to the sidecar
 	// +optional
 	Env []corev1.EnvVar `json:"env,omitempty"`
+
+	// The retentionCheckInterval defines the frequency at which the
+	// system checks and enforces retention policies.
+	// +kubebuilder:default:=1800
+	// +optional
+	RetentionPolicyIntervalSeconds int `json:"retentionPolicyIntervalSeconds,omitempty"`
 }
 
 // ObjectStoreSpec defines the desired state of ObjectStore.
 type ObjectStoreSpec struct {
+	// The configuration for the barman-cloud tool suite
 	Configuration barmanapi.BarmanObjectStoreConfiguration `json:"configuration"`
 
+	// RetentionPolicy is the retention policy to be used for backups
+	// and WALs (i.e. '60d'). The retention policy is expressed in the form
+	// of `XXu` where `XX` is a positive integer and `u` is in `[dwm]` -
+	// days, weeks, months.
+	// +kubebuilder:validation:Pattern=^[1-9][0-9]*[dwm]$
+	// +optional
+	RetentionPolicy string `json:"retentionPolicy,omitempty"`
+
 	// +optional
 	InstanceSidecarConfiguration InstanceSidecarConfiguration `json:"instanceSidecarConfiguration,omitempty"`
 }
 
 // ObjectStoreStatus defines the observed state of ObjectStore.
 type ObjectStoreStatus struct {
-	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
-	// Important: Run "make" to regenerate code after modifying this file
+	// ServerRecoveryWindow maps each server to its recovery window
+	ServerRecoveryWindow map[string]RecoveryWindow `json:"serverRecoveryWindow,omitempty"`
+}
+
+// RecoveryWindow represents the time span between the first
+// recoverability point and the last successful backup of a PostgreSQL
+// server, defining the period during which data can be restored.
+type RecoveryWindow struct {
+	// The first recoverability point in a PostgreSQL server refers to
+	// the earliest point in time to which the database can be
+	// restored.
+	FirstRecoverabilityPoint *metav1.Time `json:"firstRecoverabilityPoint,omitempty"`
+
+	// The last successful backup time
+	LastSuccessfulBackupTime *metav1.Time `json:"lastSuccussfulBackupTime,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/v1/zz_generated.deepcopy.go

@@ -40,9 +40,7 @@ spec:
             description: ObjectStoreSpec defines the desired state of ObjectStore.
             properties:
               configuration:
-                description: |-
-                  BarmanObjectStoreConfiguration contains the backup configuration
-                  using Barman against an S3-compatible object storage
+                description: The configuration for the barman-cloud tool suite
                 properties:
                   azureCredentials:
                     description: The credentials to use to upload data to Azure Blob
@@ -502,12 +500,49 @@ spec:
                       - name
                       type: object
                     type: array
+                  retentionPolicyIntervalSeconds:
+                    default: 1800
+                    description: |-
+                      The retentionCheckInterval defines the frequency at which the
+                      system checks and enforces retention policies.
+                    type: integer
                 type: object
+              retentionPolicy:
+                description: |-
+                  RetentionPolicy is the retention policy to be used for backups
+                  and WALs (i.e. '60d'). The retention policy is expressed in the form
+                  of `XXu` where `XX` is a positive integer and `u` is in `[dwm]` -
+                  days, weeks, months.
+                pattern: ^[1-9][0-9]*[dwm]$
+                type: string
             required:
             - configuration
             type: object
           status:
             description: ObjectStoreStatus defines the observed state of ObjectStore.
+            properties:
+              serverRecoveryWindow:
+                additionalProperties:
+                  description: |-
+                    RecoveryWindow represents the time span between the first
+                    recoverability point and the last successful backup of a PostgreSQL
+                    server, defining the period during which data can be restored.
+                  properties:
+                    firstRecoverabilityPoint:
+                      description: |-
+                        The first recoverability point in a PostgreSQL server refers to
+                        the earliest point in time to which the database can be
+                        restored.
+                      format: date-time
+                      type: string
+                    lastSuccussfulBackupTime:
+                      description: The last successful backup time
+                      format: date-time
+                      type: string
+                  type: object
+                description: ServerRecoveryWindow maps each server to its recovery
+                  window
+                type: object
             type: object
         required:
         - metadata

config/crd/bases/barmancloud.cnpg.io_objectstores.yaml

@@ -0,0 +1,29 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: cluster-example
+spec:
+  instances: 3
+  imagePullPolicy: Always
+
+  backup:
+    barmanObjectStore:
+      destinationPath: s3://backups/
+      endpointURL: http://minio:9000
+      s3Credentials:
+        accessKeyId:
+          name: minio
+          key: ACCESS_KEY_ID
+        secretAccessKey:
+          name: minio
+          key: ACCESS_SECRET_KEY
+      wal:
+        compression: gzip
+      data:
+        additionalCommandArgs:
+          - "--min-chunk-size=5MB"
+          - "--read-timeout=60"
+          - "-vv"
+
+  storage:
+    size: 1Gi

docs/examples/cluster-example-legacy.yaml

@@ -3,6 +3,9 @@ kind: ObjectStore
 metadata:
   name: minio-store
 spec:
+  retentionPolicy: "1m"
+  instanceSidecarConfiguration:
+    retentionPolicyIntervalSeconds: 30
   configuration:
     endpointCA:
       name: minio-server-tls

docs/examples/minio-store.yaml

@@ -0,0 +1,14 @@
+#!/usr/bin/env sh
+
+# This script builds the images of the barman cloud plugin, to be used
+# to quickly test images in a development environment.
+#
+# After each run, the built images will have these names:
+#
+# - `plugin-barman-cloud:dev`
+# - `plugin-barman-cloud-sidecar:dev`
+
+set -eu
+
+docker build -t plugin-barman-cloud:dev --file containers/Dockerfile.plugin .
+docker build -t plugin-barman-cloud-sidecar:dev --file containers/Dockerfile.sidecar .

hack/build-dev-image.sh

@@ -18,6 +18,7 @@ func NewCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			requiredSettings := []string{
 				"namespace",
+				"cluster-name",
 				"pod-name",
 				"spool-directory",
 			}
@@ -33,6 +34,7 @@ func NewCmd() *cobra.Command {
 	}
 
 	_ = viper.BindEnv("namespace", "NAMESPACE")
+	_ = viper.BindEnv("cluster-name", "CLUSTER_NAME")
 	_ = viper.BindEnv("pod-name", "POD_NAME")
 	_ = viper.BindEnv("pgdata", "PGDATA")
 	_ = viper.BindEnv("spool-directory", "SPOOL_DIRECTORY")

internal/cmd/instance/main.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"strconv"
 	"time"
 
 	barmanBackup "github.com/cloudnative-pg/barman-cloud/pkg/backup"
@@ -20,7 +19,6 @@ import (
 
 	barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1"
 	"github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/common"
-	"github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/metadata"
 	"github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/operator/config"
 )
 
@@ -148,11 +146,6 @@ func (b BackupServiceImplementation) Backup(
 		EndLsn:     executedBackupInfo.EndLSN,
 		InstanceId: b.InstanceName,
 		Online:     true,
-		Metadata: map[string]string{
-			"timeline":    strconv.Itoa(executedBackupInfo.TimeLine),
-			"version":     metadata.Data.Version,
-			"name":        metadata.Data.Name,
-			"displayName": metadata.Data.DisplayName,
-		},
+		Metadata:   newBackupResultMetadata(configuration.Cluster.ObjectMeta.UID, executedBackupInfo.TimeLine).toMap(),
 	}, nil
 }

internal/cnpgi/instance/backup.go

@@ -5,15 +5,16 @@ import (
 	"path"
 
 	cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
+	"github.com/cloudnative-pg/machinery/pkg/log"
 	"github.com/spf13/viper"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/scheme"
 
 	barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1"
@@ -26,7 +27,10 @@ func Start(ctx context.Context) error {
 
 	setupLog := log.FromContext(ctx)
 	setupLog.Info("Starting barman cloud instance plugin")
+
 	podName := viper.GetString("pod-name")
+	clusterName := viper.GetString("cluster-name")
+	namespace := viper.GetString("namespace")
 
 	controllerOptions := ctrl.Options{
 		Scheme: scheme,
@@ -36,6 +40,7 @@ func Start(ctx context.Context) error {
 					&corev1.Secret{},
 					&barmancloudv1.ObjectStore{},
 					&cnpgv1.Cluster{},
+					&cnpgv1.Backup{},
 				},
 			},
 		},
@@ -47,10 +52,11 @@ func Start(ctx context.Context) error {
 		return err
 	}
 
+	customCacheClient := extendedclient.NewExtendedClient(mgr.GetClient())
+
 	if err := mgr.Add(&CNPGI{
-		Client:       extendedclient.NewExtendedClient(mgr.GetClient()),
-		InstanceName: podName,
-		// TODO: improve
+		Client:         customCacheClient,
+		InstanceName:   podName,
 		PGDataPath:     viper.GetString("pgdata"),
 		PGWALPath:      path.Join(viper.GetString("pgdata"), "pg_wal"),
 		SpoolDirectory: viper.GetString("spool-directory"),
@@ -60,6 +66,19 @@ func Start(ctx context.Context) error {
 		return err
 	}
 
+	if err := mgr.Add(&CatalogMaintenanceRunnable{
+		Client:   customCacheClient,
+		Recorder: mgr.GetEventRecorderFor("policy-runnable"),
+		ClusterKey: types.NamespacedName{
+			Namespace: namespace,
+			Name:      clusterName,
+		},
+		CurrentPodName: podName,
+	}); err != nil {
+		setupLog.Error(err, "unable to policy enforcement runnable")
+		return err
+	}
+
 	if err := mgr.Start(ctx); err != nil {
 		return err
 	}