Merge pull request #492 from vshn/feature/cnpg-backup

Feature: CNPG backup

Author: vshnbot

pkg/comp-functions/functions/common/backup/backup.go

@@ -37,7 +37,7 @@ func AddK8upBackup(ctx context.Context, svc *runtime.ServiceRuntime, comp common
 
 	// Always create/preserve the backup bucket (handles both enabled and disabled states)
 	l.Info("Creating/preserving backup bucket", "backupEnabled", comp.IsBackupEnabled())
-	err := createObjectBucket(ctx, comp, svc)
+	err := CreateObjectBucket(ctx, comp, svc)
 	if err != nil {
 		return fmt.Errorf("cannot create/preserve backup bucket: %w", err)
 	}
@@ -76,7 +76,8 @@ func AddK8upBackup(ctx context.Context, svc *runtime.ServiceRuntime, comp common
 	return nil
 }
 
-func createObjectBucket(ctx context.Context, comp common.InfoGetter, svc *runtime.ServiceRuntime) error {
+// Create object bucket for backups
+func CreateObjectBucket(ctx context.Context, comp common.InfoGetter, svc *runtime.ServiceRuntime) error {
 	l := controllerruntime.LoggerFrom(ctx)
 
 	if comp.GetName() == "" {

pkg/comp-functions/functions/vshnpostgrescnpg/backup.go

@@ -0,0 +1,109 @@
+package vshnpostgrescnpg
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"strings"
+
+	vshnv1 "github.com/vshn/appcat/v4/apis/vshn/v1"
+	"github.com/vshn/appcat/v4/pkg/comp-functions/functions/common"
+	"github.com/vshn/appcat/v4/pkg/comp-functions/functions/common/backup"
+	"github.com/vshn/appcat/v4/pkg/comp-functions/runtime"
+)
+
+// Backup bucket connection details
+type backupCredentials struct {
+	endpoint  string
+	bucket    string
+	region    string
+	accessId  string
+	accessKey string
+}
+
+// Bootstrap backup (if enabled)
+func SetupBackup(ctx context.Context, svc *runtime.ServiceRuntime, comp *vshnv1.VSHNPostgreSQL, values map[string]any) error {
+	// CreateObjectBucket has its own IsBackupEnabled to deal with bucket retention
+	if err := backup.CreateObjectBucket(ctx, comp, svc); err != nil {
+		return err
+	}
+
+	maintTime := common.SetRandomMaintenanceSchedule(comp)
+	common.SetRandomBackupSchedule(comp, &maintTime)
+
+	if comp.IsBackupEnabled() {
+		if err := insertBackupValues(svc, comp, values); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Add backup config to helm values
+func insertBackupValues(svc *runtime.ServiceRuntime, comp *vshnv1.VSHNPostgreSQL, values map[string]any) error {
+	connectionDetails, err := getBackupBucketConnectionDetails(svc, comp)
+	if err != nil {
+		return err
+	}
+
+	retention := comp.GetBackupRetention()
+	retentionDays := retention.KeepDaily
+	if retentionDays <= 0 {
+		retentionDays = 6
+	}
+
+	maps.Copy(values, map[string]any{
+		"backups": map[string]any{
+			"enabled":         true,
+			"endpointURL":     connectionDetails.endpoint,
+			"retentionPolicy": fmt.Sprintf("%dd", retentionDays),
+			"scheduledBackups": []map[string]string{{
+				"name":                 "default",
+				"method":               "barmanObjectStore",
+				"schedule":             transformSchedule(comp.GetBackupSchedule()),
+				"backupOwnerReference": "self",
+			}},
+			"data": map[string]string{
+				"encryption": "",
+			},
+			"wal": map[string]string{
+				"encryption": "",
+			},
+			"s3": map[string]string{
+				"bucket":    connectionDetails.bucket,
+				"region":    connectionDetails.region,
+				"accessKey": connectionDetails.accessId,
+				"secretKey": connectionDetails.accessKey,
+				// The S3 secret MUST have the keys ACCESS_KEY_ID and ACCESS_SECRET_KEY.
+				// This is currently hardcoded in the chart, and as the CD secret does not have those keys in verbatim,
+				// we are forced to pass those values to the chart and letting it create its own secret instead.
+			},
+		},
+	})
+
+	return nil
+}
+
+func getBackupBucketConnectionDetails(svc *runtime.ServiceRuntime, comp *vshnv1.VSHNPostgreSQL) (backupCredentials, error) {
+	backupCredentials := backupCredentials{}
+	cd, err := svc.GetObservedComposedResourceConnectionDetails(comp.GetName() + "-backup")
+	if err != nil && err == runtime.ErrNotFound {
+		return backupCredentials, fmt.Errorf("backup bucket connection details not found")
+	} else if err != nil {
+		return backupCredentials, err
+	}
+
+	endpoint, _ := strings.CutSuffix(string(cd["ENDPOINT_URL"]), "/")
+	backupCredentials.endpoint = endpoint
+	backupCredentials.bucket = string(cd["BUCKET_NAME"])
+	backupCredentials.region = string(cd["AWS_REGION"])
+	backupCredentials.accessId = string(cd["AWS_ACCESS_KEY_ID"])
+	backupCredentials.accessKey = string(cd["AWS_SECRET_ACCESS_KEY"])
+	return backupCredentials, nil
+}
+
+// Transform backup schedule according to robfig/cron (used by CNPG)
+// https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
+func transformSchedule(thisSchedule string) string {
+	return fmt.Sprintf("0 %s", thisSchedule)
+}

pkg/comp-functions/functions/vshnpostgrescnpg/backup_test.go

@@ -0,0 +1,69 @@
+package vshnpostgrescnpg
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	appcatv1 "github.com/vshn/appcat/v4/apis/v1"
+	"github.com/vshn/appcat/v4/pkg/comp-functions/runtime"
+)
+
+func Test_BackupBootstrapDisabled(t *testing.T) {
+	svc, comp := getPostgreSqlComp(t, "vshn-postgres/deploy/05_backup_disabled_cnpg.yaml")
+	ctx := context.TODO()
+
+	// If backup has been disabled and the instance first created, expect neither bucket nor backup values
+	values, err := createCnpgHelmValues(ctx, svc, comp)
+	assert.NoError(t, err)
+
+	assert.NoError(t, SetupBackup(ctx, svc, comp, values))
+	assert.Nil(t, values["backups"])
+
+	bucketName := comp.GetName() + "-backup"
+	err = svc.GetDesiredComposedResourceByName(&appcatv1.XObjectBucket{}, bucketName)
+	assert.ErrorIs(t, err, runtime.ErrNotFound)
+}
+
+func TestBackupBooststrapEnabled(t *testing.T) {
+	svc, comp := getPostgreSqlComp(t, "vshn-postgres/deploy/05_backup_cnpg.yaml")
+	ctx := context.TODO()
+
+	// If backup has been enabled, expect values and backup bucket
+	values, err := createCnpgHelmValues(ctx, svc, comp)
+	assert.NoError(t, err)
+
+	assert.NoError(t, SetupBackup(ctx, svc, comp, values))
+	assert.NotNil(t, values["backups"])
+
+	backupValues := values["backups"].(map[string]any)
+
+	// Enabled, retention
+	assert.True(t, backupValues["enabled"].(bool))
+	assert.Equal(t, backupValues["retentionPolicy"], "6d")
+
+	// Schedule
+	assert.Equal(t,
+		backupValues["scheduledBackups"].([]map[string]string)[0]["schedule"],
+		transformSchedule(comp.GetBackupSchedule()),
+	)
+
+	// Bucket configuration
+	cd, err := getBackupBucketConnectionDetails(svc, comp)
+	assert.NoError(t, err)
+	assert.Equal(t, cd.endpoint, "https://s3.minio.local") // No trailing /
+	assert.Equal(t, cd.bucket, "backupBucket")
+	assert.Equal(t, cd.region, "rma")
+	assert.Equal(t, cd.accessId, "secretAccessId")
+	assert.Equal(t, cd.accessKey, "secretAccessKey")
+
+	assert.Equal(t, cd.endpoint, backupValues["endpointURL"])
+	assert.Equal(t, cd.bucket, backupValues["s3"].(map[string]string)["bucket"])
+	assert.Equal(t, cd.region, backupValues["s3"].(map[string]string)["region"])
+	assert.Equal(t, cd.accessId, backupValues["s3"].(map[string]string)["accessKey"])
+	assert.Equal(t, cd.accessKey, backupValues["s3"].(map[string]string)["secretKey"])
+
+	bucketName := comp.GetName() + "-backup"
+	err = svc.GetDesiredComposedResourceByName(&appcatv1.XObjectBucket{}, bucketName)
+	assert.NoError(t, err)
+}

pkg/comp-functions/functions/vshnpostgrescnpg/deploy.go

@@ -11,9 +11,7 @@ import (
 
 	cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
 	certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
-	xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
 	xfnproto "github.com/crossplane/function-sdk-go/proto/v1"
-	appcatv1 "github.com/vshn/appcat/v4/apis/v1"
 	vshnv1 "github.com/vshn/appcat/v4/apis/vshn/v1"
 	k8sruntime "k8s.io/apimachinery/pkg/runtime"
 
@@ -141,49 +139,6 @@ func createCerts(comp *vshnv1.VSHNPostgreSQL, svc *runtime.ServiceRuntime) error
 	return nil
 }
 
-func createObjectBucket(comp *vshnv1.VSHNPostgreSQL, svc *runtime.ServiceRuntime) error {
-	xObjectBucket := &appcatv1.XObjectBucket{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: comp.GetName(),
-			Labels: map[string]string{
-				runtime.ProviderConfigIgnoreLabel: "true",
-			},
-		},
-		Spec: appcatv1.XObjectBucketSpec{
-			Parameters: appcatv1.ObjectBucketParameters{
-				BucketName: fmt.Sprintf("%s-%s-%s", comp.GetName(), svc.Config.Data["bucketRegion"], "backup"),
-			},
-			ResourceSpec: xpv1.ResourceSpec{
-				WriteConnectionSecretToReference: &xpv1.SecretReference{
-					Name:      "pgbucket-" + comp.GetName(),
-					Namespace: svc.GetCrossplaneNamespace(),
-				},
-			},
-		},
-	}
-
-	xObjectBucket.Spec.Parameters.BucketName = getBucketName(svc, xObjectBucket)
-
-	err := svc.SetDesiredComposedResourceWithName(xObjectBucket, "pg-bucket")
-	if err != nil {
-		err = fmt.Errorf("cannot create xObjectBucket: %w", err)
-		return err
-	}
-
-	return nil
-}
-
-func getBucketName(svc *runtime.ServiceRuntime, currentBucket *appcatv1.XObjectBucket) string {
-	bucket := &appcatv1.XObjectBucket{}
-
-	err := svc.GetObservedComposedResource(bucket, "pg-bucket")
-	if err != nil {
-		return currentBucket.Spec.Parameters.BucketName
-	}
-
-	return bucket.Spec.Parameters.BucketName
-}
-
 // Deploy PostgresQL using the CNPG cluster helm chart
 func deployPostgresSQLUsingCNPG(ctx context.Context, comp *vshnv1.VSHNPostgreSQL, svc *runtime.ServiceRuntime) *xfnproto.Result {
 	// Deploy
@@ -192,6 +147,10 @@ func deployPostgresSQLUsingCNPG(ctx context.Context, comp *vshnv1.VSHNPostgreSQL
 		return runtime.NewFatalResult(fmt.Errorf("cannot create helm values: %w", err))
 	}
 
+	if err := SetupBackup(ctx, svc, comp, values); err != nil {
+		return runtime.NewWarningResult(fmt.Sprintf("cannot set up backup: %v", err))
+	}
+
 	svc.Log.Info("Creating Helm release for CNPG PostgreSQL")
 	release, err := common.NewRelease(ctx, svc, comp, values, comp.GetName()+"-cnpg")
 	if err != nil {

pkg/comp-functions/functions/vshnpostgrescnpg/deploy_test.go

@@ -8,6 +8,7 @@ import (
 	vshnv1 "github.com/vshn/appcat/v4/apis/vshn/v1"
 	"github.com/vshn/appcat/v4/pkg/comp-functions/functions/commontest"
 	"github.com/vshn/appcat/v4/pkg/comp-functions/runtime"
+	"k8s.io/utils/ptr"
 )
 
 const (
@@ -19,6 +20,7 @@ func Test_deploy(t *testing.T) {
 	svc, comp := getSvcCompCnpg(t)
 	ctx := context.TODO()
 
+	comp.Spec.Parameters.Backup.Enabled = ptr.To(false)
 	assert.Nil(t, deployPostgresSQLUsingCNPG(ctx, comp, svc))
 }