feat: Differential backup implementation (#243)

* feat: Differential backup implementation for postgres

* feat: Added minio backup/restore in mirror mode (#244)

* fix: Adjusted properties (#245)

* fix: Change order to allow worker join

* fix: use separate name for github token

* fix: Drop section

Author: adskyiproger

.github/workflows/get-secret-from-environment.yml

@@ -31,6 +31,10 @@ on:
         required: false
       BACKUP_SERVER_USER:
         required: false
+      POSTGRES_USER:
+        required: false
+      POSTGRES_PASSWORD:
+        required: false
 
 jobs:
   check-environment:

.github/workflows/github-to-k8s-sync-env.yml

@@ -32,20 +32,25 @@ jobs:
   get-restore-env-name:
     name: Get restore environment name
     outputs:
-      restore_env_name: ${{ steps.set-restore-env.outputs.restore-env-name }}
+      restore_env_name: ${{ vars.RESTORE_ENVIRONMENT_NAME }}
     runs-on: ubuntu-24.04
     environment: ${{ inputs.environment }}
     steps:
-      - name: Set restore environment name
-        id: set-restore-env
+      - name: "Env name: ${{ vars.RESTORE_ENVIRONMENT_NAME }}"
         run: |
-          if [ "${{ vars.RESTORE_ENVIRONMENT_NAME }}" != "false" ]; then
-          echo "restore-env-name=${{ vars.RESTORE_ENVIRONMENT_NAME }}"
-          echo "restore-env-name=${{ vars.RESTORE_ENVIRONMENT_NAME }}" >> $GITHUB_OUTPUT
-          fi
-      - name: "Env name: ${{ steps.set-restore-env.outputs.restore-env-name }}"
+          echo "Determined restore environment name: ${{ vars.RESTORE_ENVIRONMENT_NAME }}"
+  get-restore-env-mode:
+    needs: get-restore-env-name
+    if: needs.get-restore-env-name.outputs.restore_env_name
+    name: Get restore environment mode
+    outputs:
+      restore_mode: ${{ vars.BACKUP_ENVIRONMENT_MODE }}
+    runs-on: ubuntu-24.04
+    environment: ${{ needs.get-restore-env-name.outputs.restore_env_name }}
+    steps:
+      - name: "Env name: ${{ needs.get-restore-env-name.outputs.restore_env_name }} and mode ${{ vars.BACKUP_ENVIRONMENT_MODE }}"
         run: |
-          echo "Determined restore environment name: ${{ steps.set-restore-env.outputs.restore-env-name }}"
+          echo "Determined restore environment mode: ${{ vars.BACKUP_ENVIRONMENT_MODE }}"
 
   get-restore-encryption-key:
     needs: get-restore-env-name
@@ -71,7 +76,34 @@ jobs:
       gh_token: ${{ secrets.GH_TOKEN }}
       encryption_key: ${{ secrets.GH_ENCRYPTION_PASSWORD }}
       BACKUP_HOST_PRIVATE_KEY: ${{ secrets.BACKUP_HOST_PRIVATE_KEY }}
-
+  get-restore-postgres-admin-user:
+    needs:
+      - get-restore-env-name
+      - get-restore-env-mode
+    if: needs.get-restore-env-name.outputs.restore_env_name && needs.get-restore-env-mode.outputs.restore_mode == 'differential'
+    name: Get postgres admin user
+    uses: ./.github/workflows/get-secret-from-environment.yml
+    with:
+      secret_name: 'POSTGRES_USER'
+      env_name: ${{ needs.get-restore-env-name.outputs.restore_env_name }}
+    secrets:
+      gh_token: ${{ secrets.GH_TOKEN }}
+      encryption_key: ${{ secrets.GH_ENCRYPTION_PASSWORD }}
+      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+  get-restore-postgres-admin-password:
+    needs:
+      - get-restore-env-name
+      - get-restore-env-mode
+    if: needs.get-restore-env-name.outputs.restore_env_name && needs.get-restore-env-mode.outputs.restore_mode == 'differential'
+    name: Get postgres admin password
+    uses: ./.github/workflows/get-secret-from-environment.yml
+    with:
+      secret_name: 'POSTGRES_PASSWORD'
+      env_name: ${{ needs.get-restore-env-name.outputs.restore_env_name }}
+    secrets:
+      gh_token: ${{ secrets.GH_TOKEN }}
+      encryption_key: ${{ secrets.GH_ENCRYPTION_PASSWORD }}
+      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
   get-restore-host:
     needs: get-restore-env-name
     if: needs.get-restore-env-name.outputs.restore_env_name
@@ -101,6 +133,8 @@ jobs:
       - get-restore-host
       - get-restore-ssh-user
       - get-restore-encryption-key
+      - get-restore-postgres-admin-user
+      - get-restore-postgres-admin-password
     if: always()
     name: Sync GitHub env to Kubernetes
     environment: ${{ inputs.environment }}
@@ -144,10 +178,20 @@ jobs:
           openssl enc -aes-256-cbc -pbkdf2 -d -salt -k "${{ secrets.GH_ENCRYPTION_PASSWORD }}" | base64)
           echo "::add-mask::$BACKUP_SERVER_USER"
 
+          POSTGRES_USER=$(echo "${{ needs.get-restore-postgres-admin-user.outputs.secret_value }}" | base64 --decode | \
+          openssl enc -aes-256-cbc -pbkdf2 -d -salt -k "${{ secrets.GH_ENCRYPTION_PASSWORD }}" | base64)
+          echo "::add-mask::$POSTGRES_USER"
+
+          POSTGRES_PASSWORD=$(echo "${{ needs.get-restore-postgres-admin-password.outputs.secret_value }}" | base64 --decode | \
+          openssl enc -aes-256-cbc -pbkdf2 -d -salt -k "${{ secrets.GH_ENCRYPTION_PASSWORD }}" | base64)
+          echo "::add-mask::$POSTGRES_PASSWORD"
+
           echo RESTORE_ENCRYPTION_PASSPHRASE=$RESTORE_ENCRYPTION_PASSPHRASE >> $env_file
           echo BACKUP_HOST=$BACKUP_HOST >> $env_file
           echo BACKUP_HOST_PRIVATE_KEY=$BACKUP_HOST_PRIVATE_KEY >> $env_file
           echo BACKUP_SERVER_USER=$BACKUP_SERVER_USER >> $env_file
+          echo POSTGRES_USER=$POSTGRES_USER >> $env_file
+          echo POSTGRES_PASSWORD=$POSTGRES_PASSWORD >> $env_file
           grep BACKUP_HOST_PRIVATE_KEY $env_file || echo "No restore encryption passphrase to add"
 
       - name: Preprocess mapping into Secret YAMLs
@@ -185,7 +229,7 @@ jobs:
                 k8s_var="$key"
               fi
 
-              value=$(grep "^$github_var=" "$env_file" | cut -d= -f2- || true)
+              value=$(grep "^$github_var=" "$env_file" | cut -d= -f2- | tail -1 || true)
 
               if [ -n "$value" ]; then
                 echo "  $k8s_var: $value"

charts/dependencies/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: opencrvs-dependencies-chart
 description: Dependencies required by OpenCRVS
 type: application
-version: 0.2.7
+version: 0.2.9
 appVersion: v1.9.10

charts/dependencies/README.md

@@ -73,8 +73,22 @@ This section allows you to configure the postgres deployment within your infrast
 | storage_type    | string | `pvc` |  Kubernetes storage type, available options are `pvc` or `host_path`. More information are at [Storage Configuration](#storage-configuration) |
 | host_data_path  | string | `/data/postgres` | Path to persistent data on VM (host) |
 | node_selector   | dict | `{}` | Label selector for datastore nodes, usually used to keep data persistent |
-| backup_schedule | string | `n/a` | Backup cronjob schedule, if not defined then values from `backup.schedule` is used |
-| backup_server_dir | string | `n/a` | Directory to store encrypted backup on backup server, if not defined `backup.backup_server_dir` is used |
+| backup.{}       | dict | `{}` | Backup configuration section, for more information please check `values.yaml` and **Backup section** in this README |
+| backup.enabled  | string | `false` | Backup enabled or disabled, section has higher priority over global `backup` section |
+| backup.type     | string | `dump` | `dump` is a full logical database dump, `differential` is a physical backup using pgBackRest |
+| backup.server_secret | string | `backup-server-ssh-credentials` | Name of the Kubernetes secret with backup server credentials       |
+| backup.encryption_secret | string | `backup-encryption-secret` | Name of the Kubernetes secret containing the backup encryption key |
+| backup.schedule | dict | `{}` | Backup cronjob schedule |
+| backup.schedule.dump | string | `0 1 * * *` | Used only when type=dump, if not defined then value from `backup.schedule` is used |
+| backup.schedule.full | string | `0 1 * * 0` | Full backup schedule. Used when type=differential, note that value from `backup.schedule` is ignored |
+| `backup.schedule.differential` | string | `0 1 * * 1-6` | Differential backup schedule. Used when type=differential, note that value from `backup.schedule` is ignored |
+| backup.server_dir | string | `n/a` | Directory to store encrypted backup on backup server, if not defined `backup.backup_server_dir` is used |
+| restore.{}       | dict | `{}` | Restore configuration section, for more information please check `values.yaml` and **Restore section** in this README |
+| restore.enabled  | string | `false` | Restore enabled or disabled, section has higher priority over global `restore` section |
+| restore.server_secret | string | `backup-server-ssh-credentials` | Name of the Kubernetes secret with backup server credentials, usually backup server is used for restore, thats why credentials are shared |
+| restore.encryption_secret | string | `restore-encryption-secret` | Name of the Kubernetes secret containing the backup encryption key |
+| restore.schedule | string | `0 3 * * *` | Restore cronjob schedule, if not defined then value from `restore.schedule` is used |
+
 
 ## Elasticsearch
 
@@ -89,11 +103,11 @@ This section allows you to configure the deployment and authentication settings
 | storage_type    | string | `pvc` |  Kubernetes storage type, available options are `pvc` or `host_path`. More information are at [Storage Configuration](#storage-configuration) |
 | host_data_path  | string | `/data/elasticsearch` | Path to persistent data on VM (host) |
 | node_selector   | dict | `{}` | Label selector for datastore nodes, usually used to keep data persistent |
-| backup_schedule | string | `n/a` | Backup cronjob schedule, if not defined then values from `backup.schedule` is used |
-| backup_server_dir | string | `n/a` | Directory to store encrypted backup on backup server, if not defined `backup.backup_server_dir` is used |
+
 
 ## MinIO
 
+### Configuration options
 | Key | Default value | Description |
 |-|-|-|
 | enabled | true | Enable or disable minio service |
@@ -102,9 +116,19 @@ This section allows you to configure the deployment and authentication settings
 | storage_type    | string | `pvc` |  Kubernetes storage type, available options are `pvc` or `host_path`. More information are at [Storage Configuration](#storage-configuration) |
 | host_data_path  | string | `/data/minio` | Path to persistent data on VM (host) |
 | node_selector   | dict | `{}` | Label selector for datastore nodes, usually used to keep data persistent |
-| backup_schedule | string | `n/a` | Backup cronjob schedule, if not defined then values from `backup.schedule` is used |
-| backup_server_dir | string | `n/a` | Directory to store encrypted backup on backup server, if not defined `backup.backup_server_dir` is used |
-
+| backup.{}       | dict | `{}` | Backup configuration section, for more information please check `values.yaml` and **Backup section** in this README |
+| backup.enabled  | string | `false` | Backup enabled or disabled, section has higher priority over global `backup` section |
+| backup.type     | string | `dump` | `dump` is a full filesystem dump, `differential` is rsync from MinIO filesystem on remote backup server |
+| backup.server_secret | string | `backup-server-ssh-credentials` | Name of the Kubernetes secret with backup server credentials       |
+| backup.schedule | string | `0 1 * * *` | Time to run backup job, if not defined then value from `backup.schedule` is used |
+| backup.server_dir | string | `n/a` | Directory on backup server for encrypted archive backups or filesystem rsync. Uses global value if not set |
+| restore.{}       | dict | `{}` | Restore configuration section, for more information please check `values.yaml` and **Restore section** in this README |
+| restore.enabled  | string | `false` | Enables restore functionality; section overrides global `restore` settings. |
+| restore.type     | string | `dump` | Restore method: `dump` (from encrypted archive) or `differential` (same as for backup) |
+| restore.server_secret | string | `backup-server-ssh-credentials` | Name of the Kubernetes secret with backup server credentials, usually backup server is used for restore, thats why credentials are shared |
+| restore.schedule | string | `0 3 * * *` | Restore cronjob schedule, if not defined then value from `restore.schedule` is used |
+
+### MinIO Credentials
 Setting `use_default_credentials` to `false` will generate strong password for MinIO.
 
 MinIO defaults to minioadmin and minioadmin as the access key and secret key respectively.
@@ -141,6 +165,10 @@ documents:
       - MINIO_SECRET_KEY
 ```
 
+### Backup and Restore Section Reference
+
+For detailed configuration, review the values.yaml file and refer to the Backup and Restore sections of this README.
+Adjust schedules, server credentials, and directories as needed for your deployment.
 
 ## Redis
 

charts/dependencies/files/minio/backup.sh

@@ -24,7 +24,6 @@ REMOTE_DIR="${BACKUP_REMOTE_DIR:-"/home/$BACKUP_USER"}/$BACKUP_DATE"
 # Number of retries for backup creation
 MAX_RETRIES=10
 
-BACKUP_MODE=${BACKUP_MODE:-"fs"}
 # Install required tools
 apk add --no-cache bash curl openssl openssh jq rsync minio-client
 
@@ -33,9 +32,9 @@ if [ -z "$ENCRYPT_PASS" ]; then
   echo "[$(date +%F\ %H:%M:%S)] [ERROR] Must provide ENCRYPT_PASS environment variable"
   exit 1
 fi
+
 # Mirror data before backup
-# Works well on any minio installation but is slower and requires additional disk space
-backup_mirror(){
+backup_and_archive(){
   MINIO_ALIAS=local
   mcli alias set $MINIO_ALIAS http://minio:3535 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD
 
@@ -55,13 +54,6 @@ backup_mirror(){
   cd $BACKUP_PATH && tar -zcvf $ARCHIVE_PATH .
 }
 
-# Backup entire filesystem
-# Works well on single node installation and is fastest way to create backup
-backup_fs(){
-  echo "[$(date +%F\ %H:%M:%S)] Archive backup at $ARCHIVE_PATH"
-  cd /data && tar -zcvf $ARCHIVE_PATH .
-}
-
 create_encrypted_backup(){
   echo "[$(date +%F\ %H:%M:%S)] Encrypt backup at $ARCHIVE_PATH"
   openssl enc -aes-256-cbc -pbkdf2 -salt -in "$ARCHIVE_PATH" -out "${ARCHIVE_PATH}.enc" -pass env:ENCRYPT_PASS
@@ -88,11 +80,8 @@ echo "[$(date +%F\ %H:%M:%S)] Running backup container"
 echo "[$(date +%F\ %H:%M:%S)] Setup connection to container http://minio:3535"
 
 
-if [ $BACKUP_MODE == "fs" ]; then
-  backup_fs
-elif [ $BACKUP_MODE == "mirror" ]; then
-  backup_mirror
-fi
+backup_and_archive
+
 create_encrypted_backup
 
 transfer_to_backup_host

charts/dependencies/files/minio/restore.sh

@@ -27,8 +27,6 @@ ARCHIVE_PATH="/tmp/$ARCHIVE_NAME"
 
 # Remote directory on backup server
 REMOTE_DIR="${BACKUP_REMOTE_DIR:-"/home/$BACKUP_USER"}/$RESTORE_DATE"
-# Default is filesystem restore, can be "mirror"
-RESTORE_MODE=${RESTORE_MODE:-"fs"}
 
 if [ -z "$ENCRYPT_PASS" ]; then
   echo "[$(date +%F\ %H:%M:%S)] [ERROR] Must provide ENCRYPT_PASS environment variable"
@@ -44,15 +42,7 @@ decrypt_backup() {
   echo "[$(date +%F\ %H:%M:%S)] Decrypted archive at $ARCHIVE_PATH"
 }
 
-# Restore files from filesystem backup
-restore_fs() {
-  echo "[$(date +%F\ %H:%M:%S)] Restoring MinIO data using filesystem method"
-  rm -rf "${RESTORE_DIR:?}"/*  # Clean current contents!
-  tar -zxvf "$ARCHIVE_PATH" -C "$RESTORE_DIR"
-  echo "[$(date +%F\ %H:%M:%S)] Restore of $RESTORE_DIR complete"
-}
-
-# Step 2 (alternative). Restore using MinIO mirror (bucket by bucket)
+# Restore using MinIO mirror (bucket by bucket)
 restore_mirror() {
   MINIO_ALIAS=local-restore
   mcli alias set $MINIO_ALIAS http://minio:3535 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD
@@ -95,13 +85,6 @@ transfer_from_backup_host
 
 decrypt_backup
 
-if [ "$RESTORE_MODE" == "fs" ]; then
-  restore_fs
-elif [ "$RESTORE_MODE" == "mirror" ]; then
-  restore_mirror
-else
-  echo "[$(date +%F\ %H:%M:%S)] [ERROR] Unknown RESTORE_MODE: $RESTORE_MODE" >&2
-  exit 1
-fi
+restore_mirror
 
 echo "[$(date +%F\ %H:%M:%S)] MinIO restore process completed successfully"

charts/dependencies/files/postgres/postgres-backup-config.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# OpenCRVS is also distributed under the terms of the Civil Registration
+# & Healthcare Disclaimer located at http://opencrvs.org/license.
+#
+# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
+
+set -e
+
+
+common_config(){
+apt update -q
+apt upgrade -y -q
+apt install -y -q pgbackrest openssh-client
+# Common configuration for backup and restore
+# Temporal directory required for pushing WAL files
+echo "Create pgbackrest temp directory with correct permissions"
+mkdir -p /tmp/pgbackrest
+chown postgres:postgres /tmp/pgbackrest
+chmod 770 /tmp/pgbackrest
+
+# Skip ssh keys validation for external connections
+echo "Configure key-based access to backup server"
+{
+echo "StrictHostKeyChecking no"
+echo "UserKnownHostsFile /dev/null" 
+} > /etc/ssh/ssh_config.d/backup.conf
+
+
+# Create home directory for postgres user and push ssh keys inside
+mkdir -p /var/lib/postgresql/.ssh
+cp /root/.ssh/id_rsa /var/lib/postgresql/.ssh/id_rsa
+chown -R postgres:postgres /var/lib/postgresql/.ssh/id_rsa
+chmod 400 /var/lib/postgresql/.ssh/id_rsa
+
+}
+
+backup_mode(){
+
+if [ -f "${PGDATA}/postgresql.conf" ]
+then
+echo "Database type: $DB_TYPE. Modify ${PGDATA}/postgresql.conf"
+grep -q "pgbackrest --stanza=$PGBACKREST_STANZA" ${PGDATA}/postgresql.conf || \
+cat >> ${PGDATA}/postgresql.conf <<EOF
+
+# WAL Archive Configuration
+archive_mode = on
+archive_command = 'pgbackrest --stanza=$PGBACKREST_STANZA archive-push %p'
+archive_timeout = 60
+max_wal_senders = 3
+wal_keep_size = 1GB
+max_wal_size = 2GB
+min_wal_size = 1GB
+EOF
+else
+  echo "Warning: '${PGDATA}/postgresql.conf' not found, cannot configure backup."
+fi
+
+}
+
+restore_mode(){
+    
+# pgBackRest does restore filesystem from production including configuration files
+# Drop backup section from postgresql.conf after database restore
+if [ -f ${PGDATA}/postgresql.conf ] && [ "$DB_TYPE" == "restore" ]
+then
+  echo "Database type: $DB_TYPE. Modify ${PGDATA}/postgresql.conf: Drop archive settings."
+  grep -v 'archive_' ${PGDATA}/postgresql.conf > ${PGDATA}/postgresql.conf.tmp
+  mv ${PGDATA}/postgresql.conf.tmp ${PGDATA}/postgresql.conf
+else
+  echo "Warning: '${PGDATA}/postgresql.conf' not found, cannot clean archive settings."
+fi
+}
+
+###########################################################
+# How it works?
+# 1. standalone: database without backup or restore configured and
+#            backup/restore.type=dump.
+# 2. backup: backup.type=differential.
+#            Production database is accessible for write while backup
+#            WAL is used to play transactions made while full/diff backup
+# 3. restore: restore.type=differential.
+#            After restore drop WAL section to avoid concurrent write 
+#            operations to production repository
+###########################################################
+case "${DB_TYPE:-standalone}" in
+  standalone)
+    echo "Database type: $DB_TYPE. Additional configuration is not required" && \
+    exit 0
+    ;;
+  backup)
+    common_config
+    backup_mode
+    ;;
+  restore)
+    common_config
+    restore_mode
+    ;;
+  *)
+    log "Unknown or missing DB_TYPE: '${DB_TYPE}'. Must be standalone, backup, or restore."
+    exit 1
+    ;;
+esac
+