Skip to content

Commit c59ff01

Browse files
adskyiprogeradskyiproger
committed
feat: Add MinIO backup
1 parent 7890fe9 commit c59ff01

File tree

4 files changed

+184
-2
lines changed

4 files changed

+184
-2
lines changed

‎charts/dependencies/README.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ This section allows you to configure the deployment and authentication settings
7474
| data_storage_size | string | 5Gi | Persistent volume claim size for Elasticsearch data volume |
7575
| backup_storage_size | string | 1Gi | Persistent volume claim size for Elasticsearch backup volume |
7676
| backup_schedule | string | `n/a` | Backup cronjob schedule, if not defined then values from `backup.schedule` is used |
77-
| backup_server_dir | string | `n/a` | Directory to store elasticsearch encrypted backup on backup server, if not defined `backup.backup_server_dir` is used |
77+
| backup_server_dir | string | `n/a` | Directory to store encrypted backup on backup server, if not defined `backup.backup_server_dir` is used |
7878

7979
## MinIO
8080

@@ -83,6 +83,8 @@ This section allows you to configure the deployment and authentication settings
8383
| enabled | true | Enable or disable minio service |
8484
| use_default_credentials | true | Default credentials for MinIO are username `minioadmin` and password `minioadmin`. |
8585
| data_storage_size | string | 1Gi | Persistent volume claim size for MinIO data volume |
86+
| backup_schedule | string | `n/a` | Backup cronjob schedule, if not defined then values from `backup.schedule` is used |
87+
| backup_server_dir | string | `n/a` | Directory to store encrypted backup on backup server, if not defined `backup.backup_server_dir` is used |
8688

8789
Setting `use_default_credentials` to `false` will generate strong password for MinIO.
8890

‎charts/dependencies/files/minio/backup.sh‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
#!/usr/bin/env bash
2+
3+
# This Source Code Form is subject to the terms of the Mozilla Public
4+
# License, v. 2.0. If a copy of the MPL was not distributed with this
5+
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
6+
#
7+
# OpenCRVS is also distributed under the terms of the Civil Registration
8+
# & Healthcare Disclaimer located at http://opencrvs.org/license.
9+
#
10+
# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
11+
12+
# Initial variables configuration
13+
# Today's date is used for filenames if LABEL is not provided
14+
BACKUP_DATE=$(date +%Y-%m-%d)
15+
# Local directory inside container where minio data is mounted
16+
BACKUP_DIR="/data"
17+
# Temporal directory inside container to store data
18+
BACKUP_PATH=/tmp/minio-backup
19+
mkdir -p $BACKUP_PATH
20+
# Temporal archive path inside container
21+
ARCHIVE_PATH="/tmp/minio_backup_${BACKUP_DATE}.tar.gz"
22+
# Remote directory on backup server
23+
REMOTE_DIR="${BACKUP_REMOTE_DIR:-"/home/$BACKUP_USER"}/$BACKUP_DATE"
24+
# Number of retries for backup creation
25+
MAX_RETRIES=10
26+
# Backup encryption password
27+
ENCRYPT_PASS=${ENCRYPT_PASS:?Must provide ENCRYPT_PASS}
28+
29+
BACKUP_MODE=${BACKUP_MODE:-"fs"}
30+
# Install required tools
31+
apk add --no-cache bash curl openssl openssh jq rsync minio-client
32+
33+
# Mirror data before backup
34+
# Works well on any minio installation but is slower and requires additional disk space
35+
backup_mirror(){
36+
MINIO_ALIAS=local
37+
mcli alias set $MINIO_ALIAS http://minio:3535 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD
38+
39+
# Figure out buckets to back up
40+
if [ -z "$BUCKETS" ]; then
41+
# All buckets
42+
BUCKETS=$(mcli ls --json $MINIO_ALIAS | jq -r .key | sed 's:/$::')
43+
fi
44+
echo "Backing up MinIO buckets: $BUCKETS"
45+
echo "Destination: $BACKUP_PATH"
46+
for bucket in $BUCKETS; do
47+
echo "Backing up bucket: $bucket"
48+
mcli mirror --overwrite $MINIO_ALIAS/$bucket "$BACKUP_PATH/$bucket"
49+
done
50+
51+
echo "Backup completed! Buckets saved at: $BACKUP_PATH"
52+
cd $BACKUP_PATH && tar -zcvf $ARCHIVE_PATH .
53+
}
54+
55+
# Backup entire filesystem
56+
# Works well on single node installation and is fastest way to create backup
57+
backup_fs(){
58+
echo "[$(date +%F\ %H:%M:%S)] Archive backup at $ARCHIVE_PATH"
59+
cd /data && tar -zcvf $ARCHIVE_PATH .
60+
}
61+
62+
create_encrypted_backup(){
63+
echo "[$(date +%F\ %H:%M:%S)] Encrypt backup at $ARCHIVE_PATH"
64+
openssl enc -aes-256-cbc -pbkdf2 -salt -in "$ARCHIVE_PATH" -out "${ARCHIVE_PATH}.enc" -pass env:ENCRYPT_PASS
65+
rm -f "$ARCHIVE_PATH"
66+
echo "[$(date +%F\ %H:%M:%S)] Backup encrypted at ${ARCHIVE_PATH}.enc"
67+
}
68+
69+
transfer_to_backup_host(){
70+
echo "[$(date +%F\ %H:%M:%S)] Transfer backup to remote host"
71+
if rsync -avz \
72+
--rsync-path="mkdir -p $REMOTE_DIR && rsync" \
73+
-e "ssh -i /ssh/ssh_key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
74+
"${ARCHIVE_PATH}.enc" "${BACKUP_USER}@${BACKUP_HOST}:${REMOTE_DIR}/"; then
75+
echo "[$(date +%F\ %H:%M:%S)] Encrypted backup file ${ARCHIVE_PATH}.enc transferred to backup host ${BACKUP_HOST}:${REMOTE_DIR}"
76+
else
77+
echo "[ERROR] Failed to transfer file ${ARCHIVE_PATH}.enc to backup host ${BACKUP_HOST}:${REMOTE_DIR}" >&2
78+
exit 1
79+
fi
80+
81+
}
82+
83+
echo "[$(date +%F\ %H:%M:%S)] Running backup container"
84+
85+
echo "[$(date +%F\ %H:%M:%S)] Setup connection to container http://minio:3535"
86+
87+
88+
if [ $BACKUP_MODE == "fs" ]; then
89+
backup_fs
90+
elif [ $BACKUP_MODE == "mirror" ]; then
91+
backup_mirror
92+
fi
93+
create_encrypted_backup
94+
95+
transfer_to_backup_host

‎charts/dependencies/templates/minio-scripts-configmap.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: v1
2+
data:
3+
{{- range $path, $file := .Files.Glob "files/minio/*" }}
4+
{{ base $path }}: |
5+
{{ toString $file | indent 8 }}
6+
{{- end }}
7+
kind: ConfigMap
8+
metadata:
9+
labels:
10+
app: minio
11+
name: minio-scripts

‎charts/dependencies/templates/minio.yaml‎

Lines changed: 75 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,17 @@ spec:
6565
targetPort: 3536
6666
selector:
6767
app: minio
68+
{{- if .Values.backup.enabled }}
69+
---
70+
apiVersion: v1
71+
kind: ConfigMap
72+
metadata:
73+
name: minio-backup-jobs
74+
data:
75+
root: |
76+
{{ .Values.minio.backup_schedule | default .Values.backup.schedule }} /scripts/backup.sh >> /proc/1/fd/1 2>&1
77+
{{- end }}
78+
6879
---
6980
apiVersion: apps/v1
7081
kind: StatefulSet
@@ -73,11 +84,11 @@ metadata:
7384
app: minio
7485
name: minio
7586
spec:
76-
serviceName: minio
7787
replicas: 1
7888
selector:
7989
matchLabels:
8090
app: minio
91+
serviceName: minio
8192
template:
8293
metadata:
8394
labels:
@@ -135,6 +146,53 @@ spec:
135146
volumeMounts:
136147
- mountPath: /data
137148
name: minio
149+
{{- if .Values.backup.enabled }}
150+
- name: backup
151+
command: ["crond", "-f", "-L", "/var/log/cron.log"]
152+
image: "bash"
153+
env:
154+
- name: BACKUP_MODE
155+
value: "mirror"
156+
{{- if not .Values.minio.use_default_credentials }}
157+
- name: MINIO_ROOT_USER
158+
valueFrom:
159+
secretKeyRef:
160+
name: minio-opencrvs-users
161+
key: MINIO_ROOT_USER
162+
- name: MINIO_ROOT_PASSWORD
163+
valueFrom:
164+
secretKeyRef:
165+
name: minio-opencrvs-users
166+
key: MINIO_ROOT_PASSWORD
167+
{{- end }}
168+
- name: ENCRYPT_PASS
169+
valueFrom:
170+
secretKeyRef:
171+
name: {{ .Values.backup.backup_encryption_secret }}
172+
key: backup_encryption_key
173+
- name: BACKUP_USER
174+
valueFrom:
175+
secretKeyRef:
176+
name: {{ .Values.backup.backup_server_secret }}
177+
key: user
178+
- name: BACKUP_HOST
179+
valueFrom:
180+
secretKeyRef:
181+
name: {{ .Values.backup.backup_server_secret }}
182+
key: host
183+
- name: BACKUP_REMOTE_DIR
184+
value: "{{ .Values.minio.backup_server_dir | default .Values.backup.backup_server_dir }}"
185+
volumeMounts:
186+
- name: backup-ssh-key
187+
mountPath: /ssh
188+
readOnly: true
189+
- mountPath: /data
190+
name: minio
191+
- mountPath: /scripts
192+
name: minio-scripts
193+
- name: crontab
194+
mountPath: /etc/crontabs
195+
{{- end }}
138196
restartPolicy: Always
139197
volumes:
140198
- name: minio
@@ -146,4 +204,20 @@ spec:
146204
path: {{ .Values.minio.host_data_path | default "/data/minio" }}
147205
type: DirectoryOrCreate
148206
{{- end }}
207+
{{- if .Values.backup.enabled }}
208+
- name: backup-ssh-key
209+
secret:
210+
secretName: {{ .Values.backup.backup_server_secret }}
211+
defaultMode: 0400
212+
items:
213+
- key: ssh_key
214+
path: ssh_key
215+
- name: minio-scripts
216+
configMap:
217+
name: minio-scripts
218+
defaultMode: 0755
219+
- name: crontab
220+
configMap:
221+
name: minio-backup-jobs
222+
{{- end }}
149223
{{- end }}

0 commit comments

Comments
 (0)