Skip to content

Commit bb2617a

Browse files
stgracestgrace
authored
Add logical backup for azure (#2052)
* Add logical backup for azure
1 parent c756cb2 commit bb2617a

File tree

12 files changed

+74
-5
lines changed

12 files changed

+74
-5
lines changed

‎charts/postgres-operator/crds/operatorconfigurations.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,12 @@ spec:
498498
type: string
499499
pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$'
500500
default: "30 00 * * *"
501+
logical_backup_azure_storage_account_name:
502+
type: string
503+
logical_backup_azure_storage_container:
504+
type: string
505+
logical_backup_azure_storage_account_key:
506+
type: string
501507
debug:
502508
type: object
503509
properties:

‎docker/logical-backup/Dockerfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ RUN apt-get update \
1515
gnupg \
1616
gcc \
1717
libffi-dev \
18+
&& curl -sL https://aka.ms/InstallAzureCLIDeb | bash \
1819
&& pip3 install --upgrade pip \
1920
&& pip3 install --no-cache-dir awscli --upgrade \
2021
&& pip3 install --no-cache-dir gsutil --upgrade \

‎docker/logical-backup/dump.sh

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,12 @@ function compress {
4040
pigz
4141
}
4242

43+
function az_upload {
44+
PATH_TO_BACKUP=$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
45+
46+
az storage blob upload --file "$1" --account-name "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME" --account-key "$LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY" -c "$LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER" -n "$PATH_TO_BACKUP"
47+
}
48+
4349
function aws_delete_objects {
4450
args=(
4551
"--bucket=$LOGICAL_BACKUP_S3_BUCKET"
@@ -120,7 +126,7 @@ function upload {
120126
"gcs")
121127
gcs_upload
122128
;;
123-
*)
129+
"aws")
124130
aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF))
125131
aws_delete_outdated
126132
;;
@@ -174,8 +180,13 @@ for search in "${search_strategy[@]}"; do
174180
done
175181

176182
set -x
177-
dump | compress | upload
178-
[[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 ))
179-
set +x
183+
if [ "$LOGICAL_BACKUP_PROVIDER" == "az" ]; then
184+
dump | compress > /tmp/azure-backup.sql.gz
185+
az_upload /tmp/azure-backup.sql.gz
186+
else
187+
dump | compress | upload
188+
[[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 ))
189+
set +x
180190

181-
exit $ERRORCOUNT
191+
exit $ERRORCOUNT
192+
fi

‎docs/reference/operator_parameters.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -765,6 +765,15 @@ grouped under the `logical_backup` key.
765765
[reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule)
766766
into account. Default: "30 00 \* \* \*"
767767

768+
* **logical_backup_azure_storage_account_name**
769+
Storage account name used to upload logical backups to when using Azure. Default: ""
770+
771+
* **logical_backup_azure_storage_container**
772+
Storage container used to upload logical backups to when using Azure. Default: ""
773+
774+
* **logical_backup_azure_storage_account_key**
775+
Storage account key used to authenticate with Azure when uploading logical backups. Default: ""
776+
768777
## Debugging the operator
769778

770779
Options to aid debugging of the operator itself. Grouped under the `debug` key.

‎manifests/configmap.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@ data:
8787
logical_backup_s3_sse: "AES256"
8888
# logical_backup_s3_retention_time: ""
8989
logical_backup_schedule: "30 00 * * *"
90+
# logical_backup_azure_storage_account_name: ""
91+
# logical_backup_azure_storage_container: ""
92+
# logical_backup_azure_storage_account_key: ""
9093
major_version_upgrade_mode: "manual"
9194
# major_version_upgrade_team_allow_list: ""
9295
master_dns_name_format: "{cluster}.{namespace}.{hostedzone}"

‎manifests/operatorconfiguration.crd.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -496,6 +496,12 @@ spec:
496496
type: string
497497
pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$'
498498
default: "30 00 * * *"
499+
logical_backup_azure_storage_account_name:
500+
type: string
501+
logical_backup_azure_storage_container:
502+
type: string
503+
logical_backup_azure_storage_account_key:
504+
type: string
499505
debug:
500506
type: object
501507
properties:

‎manifests/postgresql-operator-default-configuration.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,9 @@ configuration:
163163
logical_backup_s3_sse: "AES256"
164164
# logical_backup_s3_retention_time: ""
165165
logical_backup_schedule: "30 00 * * *"
166+
# logical_backup_azure_storage_account_name: ""
167+
# logical_backup_azure_storage_container: ""
168+
# logical_backup_azure_storage_account_key: ""
166169
debug:
167170
debug_logging: true
168171
enable_database_access: true

‎pkg/apis/acid.zalan.do/v1/crds.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1655,6 +1655,15 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
16551655
Type: "string",
16561656
Pattern: "^(\\d+|\\*)(/\\d+)?(\\s+(\\d+|\\*)(/\\d+)?){4}$",
16571657
},
1658+
"logical_backup_azure_storage_account_name": {
1659+
Type: "string",
1660+
},
1661+
"logical_backup_azure_storage_container": {
1662+
Type: "string",
1663+
},
1664+
"logical_backup_azure_storage_account_key": {
1665+
Type: "string",
1666+
},
16581667
},
16591668
},
16601669
"debug": {

‎pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,9 @@ type OperatorLogicalBackupConfiguration struct {
227227
RetentionTime string `json:"logical_backup_s3_retention_time,omitempty"`
228228
GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"`
229229
JobPrefix string `json:"logical_backup_job_prefix,omitempty"`
230+
AzureStorageAccountName string `json:"logical_backup_azure_storage_account_name,omitempty"`
231+
AzureStorageContainer string `json:"logical_backup_azure_storage_container,omitempty"`
232+
AzureStorageAccountKey string `json:"logical_backup_azure_storage_account_key,omitempty"`
230233
}
231234

232235
// PatroniConfiguration defines configuration for Patroni

‎pkg/cluster/k8sres.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2262,6 +2262,18 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
22622262
Name: "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS",
22632263
Value: c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials,
22642264
},
2265+
{
2266+
Name: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_NAME",
2267+
Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountName,
2268+
},
2269+
{
2270+
Name: "LOGICAL_BACKUP_AZURE_STORAGE_CONTAINER",
2271+
Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageContainer,
2272+
},
2273+
{
2274+
Name: "LOGICAL_BACKUP_AZURE_STORAGE_ACCOUNT_KEY",
2275+
Value: c.OpConfig.LogicalBackup.LogicalBackupAzureStorageAccountKey,
2276+
},
22652277
// Postgres env vars
22662278
{
22672279
Name: "PG_VERSION",

0 commit comments

Comments
 (0)