Skip to content

Commit 5357062

Browse files
FxKuFxKu
authored
add logical backup retention as manifest option (#2621)
* add logical backup retention as manifest option * added unit test for logical backup envvar generation
1 parent d70cdf1 commit 5357062

File tree

8 files changed

+290
-72
lines changed

8 files changed

+290
-72
lines changed

‎charts/postgres-operator/crds/postgresqls.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,8 @@ spec:
215215
items:
216216
type: object
217217
x-kubernetes-preserve-unknown-fields: true
218+
logicalBackupRetention:
219+
type: string
218220
logicalBackupSchedule:
219221
type: string
220222
pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$'

‎docs/reference/cluster_manifest.md

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,10 +223,17 @@ These parameters are grouped directly under the `spec` key in the manifest.
223223
Determines if the logical backup of this cluster should be taken and uploaded
224224
to S3. Default: false. Optional.
225225

226+
* **logicalBackupRetention**
227+
You can set a retention time for the logical backup cron job to remove old backup
228+
files after a new backup has been uploaded. Example values are "3 days", "2 weeks", or
229+
"1 month". It takes precedence over the global `logical_backup_s3_retention_time`
230+
configuration. Currently only supported for AWS. Optional.
231+
226232
* **logicalBackupSchedule**
227233
Schedule for the logical backup K8s cron job. Please take
228234
[the reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule)
229-
into account. Optional. Default is: "30 00 \* \* \*"
235+
into account. It takes precedence over the global `logical_backup_schedule`
236+
configuration. Optional.
230237

231238
* **additionalVolumes**
232239
List of additional volumes to mount in each container of the statefulset pod.

‎manifests/complete-postgres-manifest.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ spec:
151151

152152
# run periodic backups with k8s cron jobs
153153
# enableLogicalBackup: true
154+
# logicalBackupRetention: "3 months"
154155
# logicalBackupSchedule: "30 00 * * *"
155156

156157
# maintenanceWindows:

‎manifests/postgresql.crd.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,8 @@ spec:
213213
items:
214214
type: object
215215
x-kubernetes-preserve-unknown-fields: true
216+
logicalBackupRetention:
217+
type: string
216218
logicalBackupSchedule:
217219
type: string
218220
pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$'

‎pkg/apis/acid.zalan.do/v1/crds.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
343343
},
344344
},
345345
},
346+
"logicalBackupRetention": {
347+
Type: "string",
348+
},
346349
"logicalBackupSchedule": {
347350
Type: "string",
348351
Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$",

‎pkg/apis/acid.zalan.do/v1/postgresql_type.go

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -63,23 +63,24 @@ type PostgresSpec struct {
6363
UsersWithSecretRotation []string `json:"usersWithSecretRotation,omitempty"`
6464
UsersWithInPlaceSecretRotation []string `json:"usersWithInPlaceSecretRotation,omitempty"`
6565

66-
NumberOfInstances int32 `json:"numberOfInstances"`
67-
MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"`
68-
Clone *CloneDescription `json:"clone,omitempty"`
69-
Databases map[string]string `json:"databases,omitempty"`
70-
PreparedDatabases map[string]PreparedDatabase `json:"preparedDatabases,omitempty"`
71-
SchedulerName *string `json:"schedulerName,omitempty"`
72-
NodeAffinity *v1.NodeAffinity `json:"nodeAffinity,omitempty"`
73-
Tolerations []v1.Toleration `json:"tolerations,omitempty"`
74-
Sidecars []Sidecar `json:"sidecars,omitempty"`
75-
InitContainers []v1.Container `json:"initContainers,omitempty"`
76-
PodPriorityClassName string `json:"podPriorityClassName,omitempty"`
77-
ShmVolume *bool `json:"enableShmVolume,omitempty"`
78-
EnableLogicalBackup bool `json:"enableLogicalBackup,omitempty"`
79-
LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"`
80-
StandbyCluster *StandbyDescription `json:"standby,omitempty"`
81-
PodAnnotations map[string]string `json:"podAnnotations,omitempty"`
82-
ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"`
66+
NumberOfInstances int32 `json:"numberOfInstances"`
67+
MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"`
68+
Clone *CloneDescription `json:"clone,omitempty"`
69+
Databases map[string]string `json:"databases,omitempty"`
70+
PreparedDatabases map[string]PreparedDatabase `json:"preparedDatabases,omitempty"`
71+
SchedulerName *string `json:"schedulerName,omitempty"`
72+
NodeAffinity *v1.NodeAffinity `json:"nodeAffinity,omitempty"`
73+
Tolerations []v1.Toleration `json:"tolerations,omitempty"`
74+
Sidecars []Sidecar `json:"sidecars,omitempty"`
75+
InitContainers []v1.Container `json:"initContainers,omitempty"`
76+
PodPriorityClassName string `json:"podPriorityClassName,omitempty"`
77+
ShmVolume *bool `json:"enableShmVolume,omitempty"`
78+
EnableLogicalBackup bool `json:"enableLogicalBackup,omitempty"`
79+
LogicalBackupRetention string `json:"logicalBackupRetention,omitempty"`
80+
LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"`
81+
StandbyCluster *StandbyDescription `json:"standby,omitempty"`
82+
PodAnnotations map[string]string `json:"podAnnotations,omitempty"`
83+
ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"`
8384
// MasterServiceAnnotations takes precedence over ServiceAnnotations for master role if not empty
8485
MasterServiceAnnotations map[string]string `json:"masterServiceAnnotations,omitempty"`
8586
// ReplicaServiceAnnotations takes precedence over ServiceAnnotations for replica role if not empty

‎pkg/cluster/k8sres.go

Lines changed: 71 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -2360,6 +2360,8 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1.CronJob, error) {
23602360

23612361
func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
23622362

2363+
backupProvider := c.OpConfig.LogicalBackup.LogicalBackupProvider
2364+
23632365
envVars := []v1.EnvVar{
23642366
{
23652367
Name: "SCOPE",
@@ -2378,55 +2380,6 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
23782380
},
23792381
},
23802382
},
2381-
// Bucket env vars
2382-
{
2383-
Name: "LOGICAL_BACKUP_PROVIDER",
2384-
Value: c.OpConfig.LogicalBackup.LogicalBackupProvider,
2385-
},
2386-
{
2387-
Name: "LOGICAL_BACKUP_S3_BUCKET",
2388-
Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket,
2389-
},
2390-
{
2391-
Name: "LOGICAL_BACKUP_S3_REGION",
2392-
Value: c.OpConfig.LogicalBackup.LogicalBackupS3Region,
2393-
},
2394-
{
2395-
Name: "LOGICAL_BACKUP_S3_ENDPOINT",
2396-
Value: c.OpConfig.LogicalBackup.LogicalBackupS3Endpoint,
2397-
},
2398-
{
2399-
Name: "LOGICAL_BACKUP_S3_SSE",
2400-
Value: c.OpConfig.LogicalBackup.LogicalBackupS3SSE,
2401-
},
2402-
{
2403-
Name: "LOGICAL_BACKUP_S3_RETENTION_TIME",
2404-
Value: c.OpConfig.LogicalBackup.LogicalBackupS3RetentionTime,
2405-
},
2406-
{
2407-
Name: "LOGICAL_BACKUP_S3_BUCKET_PREFIX",
2408-
Value: c.OpConfig.LogicalBackup.LogicalBackupS3BucketPrefix,
2409-
},
2410-
{
2411-
Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX",
2412-
Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())),
2413-
},
2414-
{
2415-
Name: "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS",
2416-
Value: c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials,
2417-
},
2418-
{
2419-
Name: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME",
2420-
Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountName,
2421-
},
2422-
{
2423-
Name: "LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER",
2424-
Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageContainer,
2425-
},
2426-
{
2427-
Name: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY",
2428-
Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountKey,
2429-
},
24302383
// Postgres env vars
24312384
{
24322385
Name: "PG_VERSION",
@@ -2459,19 +2412,83 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
24592412
},
24602413
},
24612414
},
2415+
// Bucket env vars
2416+
{
2417+
Name: "LOGICAL_BACKUP_PROVIDER",
2418+
Value: backupProvider,
2419+
},
2420+
{
2421+
Name: "LOGICAL_BACKUP_S3_BUCKET",
2422+
Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket,
2423+
},
2424+
{
2425+
Name: "LOGICAL_BACKUP_S3_BUCKET_PREFIX",
2426+
Value: c.OpConfig.LogicalBackup.LogicalBackupS3BucketPrefix,
2427+
},
2428+
{
2429+
Name: "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX",
2430+
Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())),
2431+
},
24622432
}
24632433

2464-
if c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID != "" {
2465-
envVars = append(envVars, v1.EnvVar{Name: "AWS_ACCESS_KEY_ID", Value: c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID})
2466-
}
2434+
switch backupProvider {
2435+
case "s3":
2436+
envVars = appendEnvVars(envVars, []v1.EnvVar{
2437+
{
2438+
Name: "LOGICAL_BACKUP_S3_REGION",
2439+
Value: c.OpConfig.LogicalBackup.LogicalBackupS3Region,
2440+
},
2441+
{
2442+
Name: "LOGICAL_BACKUP_S3_ENDPOINT",
2443+
Value: c.OpConfig.LogicalBackup.LogicalBackupS3Endpoint,
2444+
},
2445+
{
2446+
Name: "LOGICAL_BACKUP_S3_SSE",
2447+
Value: c.OpConfig.LogicalBackup.LogicalBackupS3SSE,
2448+
},
2449+
{
2450+
Name: "LOGICAL_BACKUP_S3_RETENTION_TIME",
2451+
Value: c.getLogicalBackupRetentionTime(),
2452+
}}...)
2453+
2454+
if c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID != "" {
2455+
envVars = append(envVars, v1.EnvVar{Name: "AWS_ACCESS_KEY_ID", Value: c.OpConfig.LogicalBackup.LogicalBackupS3AccessKeyID})
2456+
}
24672457

2468-
if c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey != "" {
2469-
envVars = append(envVars, v1.EnvVar{Name: "AWS_SECRET_ACCESS_KEY", Value: c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey})
2458+
if c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey != "" {
2459+
envVars = append(envVars, v1.EnvVar{Name: "AWS_SECRET_ACCESS_KEY", Value: c.OpConfig.LogicalBackup.LogicalBackupS3SecretAccessKey})
2460+
}
2461+
2462+
case "gcs":
2463+
envVars = append(envVars, v1.EnvVar{Name: "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials})
2464+
2465+
case "az":
2466+
envVars = appendEnvVars(envVars, []v1.EnvVar{
2467+
{
2468+
Name: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME",
2469+
Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountName,
2470+
},
2471+
{
2472+
Name: "LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER",
2473+
Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageContainer,
2474+
},
2475+
{
2476+
Name: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY",
2477+
Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountKey,
2478+
}}...)
24702479
}
24712480

24722481
return envVars
24732482
}
24742483

2484+
func (c *Cluster) getLogicalBackupRetentionTime() (retentionTime string) {
2485+
if c.Spec.LogicalBackupRetention != "" {
2486+
return c.Spec.LogicalBackupRetention
2487+
}
2488+
2489+
return c.OpConfig.LogicalBackup.LogicalBackupS3RetentionTime
2490+
}
2491+
24752492
// getLogicalBackupJobName returns the name; the job itself may not exists
24762493
func (c *Cluster) getLogicalBackupJobName() (jobName string) {
24772494
return trimCronjobName(fmt.Sprintf("%s%s", c.OpConfig.LogicalBackupJobPrefix, c.clusterName().Name))

0 commit comments

Comments
 (0)