fix

Author: adskyiproger

charts/dependencies/files/elasticsearch/backup.sh

@@ -8,20 +8,34 @@
 # & Healthcare Disclaimer located at http://opencrvs.org/license.
 #
 # Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
-ELASTIC_HOST=${ELASTIC_HOST:-"elasticsearch:9200"}
+
+# Initial variables configuration
 # Today's date is used for filenames if LABEL is not provided
-#-----------------------------------
 BACKUP_DATE=$(date +%Y-%m-%d)
-REMOTE_DIR="$REMOTE_DIR/${LABEL:-$BACKUP_DATE}"
+# Local directory inside container
+BACKUP_DIR="/backups"
+# Temporal archive path inside container
+ARCHIVE_PATH="/tmp/elasticsearch_backup_${LABEL}.tar.gz"
+# Remote directory on backup server
+REMOTE_DIR="$BACKUP_REMOTE_DIR/${LABEL:-$BACKUP_DATE}"
+# Number of retries for backup creation
+MAX_RETRIES=10
+# Reference to container within the same k8s pod
+ELASTIC_HOST=${ELASTIC_HOST:-"elasticsearch:9200"}
+# Backup encryption password
+ENCRYPT_PASS=${ENCRYPT_PASS:?Must provide ENCRYPT_PASS}
 
 # Install required tools
 apk add --no-cache bash curl openssl openssh jq
+
 echo "[$(date +%F\ %H:%M:%S)] Waiting for Elasticsearch container"
-sleep 3
+sleep ${TIMEOUT:-"300"}
+
 echo "[$(date +%F\ %H:%M:%S)] Running backup container"
-set +x
-MAX_RETRIES=10
-# elasticsearch:9200, is container:port in this case, not pod name or service name
+
+# Hostname for elasticsearch container
+# - password protected
+# - no-password access
 elasticsearch_host() {
   if [ ! -z ${ELASTIC_PASSWORD+x} ]; then
     echo "elastic:$ELASTIC_PASSWORD@${ELASTIC_HOST}"
@@ -30,6 +44,7 @@ elasticsearch_host() {
   fi
 }
 
+# List indices on server by patterns ocrvs-|events_
 get_target_indices() {
   curl -s "http://$(elasticsearch_host)/_cat/indices?h=index" \
     | grep -E '^(ocrvs-|events_)' \
@@ -53,11 +68,10 @@ create_elasticsearch_snapshot_repository() {
   exit 1
 }
 
-# Improved recursive backup function: replaced by loop.
 create_elasticsearch_backup() {
   local indices=$(get_target_indices)
   if [ -z "$indices" ]; then
-    echo "[$(date +%F\ %H:%M:%S)] No indices matching ocrvs-* or events-* found, skipping snapshot."
+    echo "[$(date +%F\ %H:%M:%S)] No indices matching ocrvs-* or events_* found, skipping snapshot."
     return 1
   fi
 
@@ -88,6 +102,18 @@ delete_all_snapshots() {
   done
 }
 
+create_encrypted_backup(){
+  # Tar/gzip all snapshot content
+  tar czf "$ARCHIVE_PATH" -C "$BACKUP_DIR" .
+
+  # Encrypt
+  openssl enc -aes-256-cbc -pbkdf2 -salt -in "$ARCHIVE_PATH" -out "${ARCHIVE_PATH}.enc" -pass env:ENCRYPT_PASS
+
+  # Remove plain file (optional)
+  rm -f "$ARCHIVE_PATH"
+  echo "Backup encrypted at ${ARCHIVE_PATH}.enc"
+}
+
 echo "List snapshots"
 curl -s "http://$(elasticsearch_host)/_cat/snapshots/ocrvs?h=id"
 
@@ -107,5 +133,8 @@ echo ""
 echo "[$(date +%F\ %H:%M:%S)] Backup Elasticsearch as a set of snapshot files into an elasticsearch sub folder"
 echo ""
 create_elasticsearch_backup
+
+create_encrypted_backup
+
 sleep 86400
 done

charts/dependencies/templates/backup-encryption-key.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.backup.enabled }}
+{{/* Prevent secret mutation between helm upgrades */}}
+{{- $secret := lookup "v1" "Secret" .Release.Namespace .Values.backup.backup_encryption_secret }}
+{{- if not $secret }}
+  {{- $backup_encryption_key := randAlphaNum 32 }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.backup.backup_encryption_secret }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/resource-policy": keep
+type: Opaque
+data:
+  backup_encryption_key: {{ $backup_encryption_key | b64enc }}
+{{- end }}
+{{- end }}

charts/dependencies/templates/elasticsearch.yaml

@@ -81,12 +81,13 @@ spec:
             {{- end }}
       {{- end }}
       initContainers:
+        # Change ownership to elasticsearch user and group
         - name: volume-permission-fix
           image: busybox
           command:
             - chown
             - -R
-            - "1000:0"  # Change ownership to elasticsearch user and group
+            - "1000:0"
             - /usr/share/elasticsearch/data
             - /data/backups/elasticsearch
           volumeMounts:
@@ -96,6 +97,7 @@ spec:
               name: elasticsearch-backup
       containers:
         - env:
+            # FIXME: Make fields configurable
             - name: ES_JAVA_OPTS
               value: -Xms2048m -Xmx8192m
             - name: cluster.routing.allocation.disk.threshold_enabled
@@ -148,6 +150,21 @@ spec:
                   name: {{ .Values.elasticsearch.admin_user_secret_name }}
                   key: ELASTIC_PASSWORD
           {{- end }}
+            - name: ENCRYPT_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.backup.backup_encryption_secret }}
+                  key: backup_encryption_key
+            - name: BACKUP_USER
+              valueFrom:
+                secretKeyRef:
+                  name: backup-ssh-key
+                  key: user
+            - name: BACKUP_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: backup-ssh-key
+                  key: host
           volumeMounts:
             - name: backup-ssh-key
               mountPath: /ssh
@@ -181,6 +198,9 @@ spec:
         - name: backup-ssh-key
           secret:
             secretName: {{ .Values.backup.backup_server_secret }}
+            items:
+              - key: ssh_key
+                path: id_rsa
         - name: elasticsearch-scripts
           configMap:
             name: elasticsearch-scripts

charts/dependencies/values.yaml

@@ -116,4 +116,5 @@ backup:
   # - user, backup server username
   # - host, hostname / IP of backup server
   # - ssh_key, ssh key for passwordless connection
-  backup_server_secret: backup-ssh-key
+  backup_server_secret: backup-ssh-key
+  backup_encryption_secret: backup-encryption-secret

infrastructure/server-setup/backup.yml

@@ -10,6 +10,7 @@
 - hosts: backup
   become: yes
   become_method: sudo
+  # FIXME: Should be executed only of backup host is defined
   tags: backup
   tasks:
     - name: Ensure backup user is present
@@ -60,6 +61,8 @@
       ansible.builtin.command: >
         kubectl create secret generic backup-ssh-key --namespace opencrvs-deps-{{ k8s_cluster_env }}
         --from-file=ssh_key=/tmp/id_ed25519_backup
+        --from-literal=user={{ backup_server_user }} \
+        --from-literal=host={{ hostvars['backup']['ansible_host'] }}
       args:
         removes: /tmp/id_ed25519_backup
       # If you want this to update the secret if it already exists, use kubectl apply or kubectl delete/create