[CLOUD-785] Refactor backups

.github/actions/remove_cluster/action.yml

@@ -13,7 +13,7 @@ runs:
   steps:
   - name: Delete RonDB Helmchart
     shell: bash
-    run: helm delete --namespace=${{ inputs.namespace }} ${{ inputs.helm_chart }}
+    run: helm delete --namespace=${{ inputs.namespace }} ${{ inputs.helm_chart }} || true
 
   # Can't just use delete Hook, since we want to print the logs of the Pod
   - name: Delete Helm Test Pods

files/scripts/backups/metadata_upload_kubectl.sh

@@ -6,8 +6,6 @@
 # for everything we want to back up. Root cannot be used over the network.
 set -e
 
-{{ include "rondb.createRcloneConfig" . }}
-
 kubectl exec \
     $MYSQLD_PODNAME \
     -c mysqld \
@@ -23,8 +21,8 @@ kubectl cp \
     -c mysqld $LOCAL_BACKUP_DIR
 ls -la $LOCAL_BACKUP_DIR
 
-{{ include "rondb.backups.defineJobNumberEnv" $ }}
-REMOTE_BACKUP_DIR={{ include "rondb.rcloneBackupRemoteName" . }}:{{ .Values.backups.s3.bucketName }}/{{ include "rondb.takeBackupPathPrefix" . }}/$JOB_NUMBER
+{{ include "rondb.backups.defineBackupIdEnv" $ }}
+REMOTE_BACKUP_DIR={{ include "rondb.rcloneBackupRemoteName" . }}:{{include "rondb.backups.bucketName" (dict "backupConfig" .Values.backups "global" .Values.global)}}/{{ include "rondb.takeBackupPathPrefix" . }}/$BACKUP_ID
 echo && rclone ls $REMOTE_BACKUP_DIR
 echo "Copying backup from $LOCAL_BACKUP_DIR to $REMOTE_BACKUP_DIR"
 rclone move $LOCAL_BACKUP_DIR $REMOTE_BACKUP_DIR

files/scripts/backups/native_upload_kubectl.sh

@@ -10,9 +10,9 @@ wait_pids=()
 NUM_NODE_GROUPS={{ .Values.clusterSize.numNodeGroups }}
 NUM_REPLICAS={{ .Values.clusterSize.activeDataReplicas }}
 
-{{ include "rondb.backups.defineJobNumberEnv" $ }}
-SOURCE_DIR=/home/hopsworks/data/ndb/backups/BACKUP/BACKUP-$JOB_NUMBER
-REMOTE_BACKUP_DIR={{ include "rondb.rcloneBackupRemoteName" . }}:{{ .Values.backups.s3.bucketName }}/{{ include "rondb.takeBackupPathPrefix" . }}/$JOB_NUMBER
+{{ include "rondb.backups.defineBackupIdEnv" $ }}
+SOURCE_DIR=/home/hopsworks/data/ndb/backups/BACKUP/BACKUP-$BACKUP_ID
+REMOTE_BACKUP_DIR={{ include "rondb.rcloneBackupRemoteName" . }}:{{ include "rondb.backups.bucketName" (dict "backupConfig" .Values.backups "global" .Values.global) }}/{{ include "rondb.takeBackupPathPrefix" . }}/$BACKUP_ID
 
 echo "Uploading backups from '$SOURCE_DIR' to object storage $REMOTE_BACKUP_DIR in parallel"
 for ((g = 0; g < NUM_NODE_GROUPS; g++)); do
@@ -85,3 +85,93 @@ if [ "$FAILED" = true ]; then
     exit 1
 fi
 echo ">>> Succeeded uploading all backups"
+
+{{ $configMap := include "rondb.backups.metadataStore.configMapName" . }}
+{{- if $configMap }}
+
+MAX_KEYS=5000
+MAX_SIZE_BYTES=900000
+BASE_CONFIGMAP={{ $configMap }}
+
+get_active_configmap(){
+    kubectl get cm -n {{ .Release.Namespace }} -l "app=backups-metadata,service=rondb,managed-by=cronjob,active=active" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true
+}
+
+log_stderr(){
+    echo "$*" >&2
+}
+
+create_configmap_if_missing() {
+  local name="$1"
+  if ! kubectl get configmap "$name" -n {{ .Release.Namespace }} >/dev/null 2>&1; then
+    log_stderr "Creating ConfigMap $name"
+    kubectl create configmap "$name" -n {{ .Release.Namespace }} >/dev/null 2>&1
+    kubectl label configmap "$name" -n {{ .Release.Namespace }} app=backups-metadata service=rondb managed-by=cronjob active=active --overwrite >/dev/null 2>&1
+  fi
+}
+
+rotate_if_needed() {
+  local cm="$1"
+
+  local key_count size_bytes
+  key_count=$(kubectl get configmap "$cm" -n {{ .Release.Namespace }} -o json | jq '.data | length')
+  size_bytes=$(kubectl get configmap "$cm" -n {{ .Release.Namespace }} -o json | jq -r '.data' | wc -c)
+
+  log_stderr "ConfigMap: $cm | Keys=$key_count | Size=${size_bytes}B"
+
+  if (( key_count >= MAX_KEYS )) || (( size_bytes >= MAX_SIZE_BYTES )); then
+    log_stderr "Threshold exceeded, rotating ConfigMap..."
+    local suffix next_suffix new_cm
+    suffix="${cm#$BASE_CONFIGMAP-}"
+    if [[ "$suffix" =~ ^[0-9]+$ ]]; then
+      next_suffix=$((suffix + 1))
+    else
+      next_suffix=1
+    fi
+    new_cm="${BASE_CONFIGMAP}-${next_suffix}"
+
+    # Create and label the new ConfigMap
+    create_configmap_if_missing "$new_cm"
+
+    # Remove active label from old configmap
+    kubectl label configmap "$cm" -n {{ .Release.Namespace }} active- --overwrite >/dev/null 2>&1  || true
+
+    log_stderr "Rotated to new ConfigMap: $new_cm"
+    echo "$new_cm"
+  else
+    echo "$cm"
+  fi
+}
+
+ACTIVE_CM=$(get_active_configmap)
+if [[ -z "$ACTIVE_CM" ]]; then
+  ACTIVE_CM="$BASE_CONFIGMAP"
+  echo "No active ConfigMap found. Creating $ACTIVE_CM ..."
+  create_configmap_if_missing "$ACTIVE_CM"
+fi
+
+ACTIVE_CM=$(rotate_if_needed "$ACTIVE_CM")
+
+# Build backup metadata info json
+echo "Updating backup metadata on ConfigMap $ACTIVE_CM "
+START_TS=$(stat -c %Y {{ include "rondb.backups.backupIdFile" . }} | awk '{printf "%.3f", $1}')
+END_TS=$(date +%s.%3N)
+
+DURATION_MS=$(awk -v start="$START_TS" -v end="$END_TS" 'BEGIN { printf "%.0f", (end - start) * 1000 }')
+
+START_TIME=$(date -u -d @"${START_TS%.*}" +"%Y-%m-%dT%H:%M:%S").$(printf "%03d" "${START_TS#*.}")Z
+END_TIME=$(date -u -d @"${END_TS%.*}" +"%Y-%m-%dT%H:%M:%S").$(printf "%03d" "${END_TS#*.}")Z
+
+STATE="SUCCESS"
+
+PATCH_JSON=$(cat <<EOF
+{
+  "data": {
+    "$BACKUP_ID": "{\"start_time\":\"$START_TIME\",\"end_time\":\"$END_TIME\",\"duration_ms\":$DURATION_MS,\"state\":\"$STATE\",\"path\":\"{{ include "rondb.backups.pathScheme" . }}/{{ include "rondb.backups.bucketName" (dict "backupConfig" .Values.backups "global" .Values.global) }}/{{ include "rondb.takeBackupPathPrefix" . }}/$BACKUP_ID\"}"
+  }
+}
+EOF
+)
+
+kubectl patch configmap "$ACTIVE_CM" -n {{ .Release.Namespace }} --type merge -p "$PATCH_JSON"
+{{- end }}

templates/backups/create.yaml

@@ -1,16 +1,17 @@
 # Copyright (c) 2024-2025 Hopsworks AB. All rights reserved.
 
-{{ if .Values.backups.enabled -}}
+{{ if include "rondb.backups.isEnabled" . -}}
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: create-backup
+  name: create-rondb-backup
   namespace: {{ .Release.Namespace }}
 spec:
-  schedule: {{ .Values.backups.schedule | quote}}
+  schedule: {{ include "rondb.backups.schedule" . | quote}}
   concurrencyPolicy: Forbid
   jobTemplate:
     spec:
+      backoffLimit: 0
       template:
         metadata:
           labels:
@@ -20,7 +21,7 @@ spec:
 {{- include "rondb.nodeSelector" (dict "nodeSelector" $.Values.nodeSelector.backup) | indent 10 }}
 {{- include "rondb.tolerations" (dict "tolerations" $.Values.tolerations.backup) | indent 10 }}
           serviceAccountName: rondb-backups-sa
-          restartPolicy: OnFailure
+          restartPolicy: Never
           initContainers:
 {{ include "rondb.apiInitContainer" . | indent 10 }}
           - name: backup-metadata
@@ -32,38 +33,23 @@ spec:
             - |
 {{ tpl (.Files.Get "files/scripts/backups/metadata_upload_kubectl.sh") . | indent 14 }}
             env:
-            - name: JOB_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
             - name: MYSQLD_PODNAME
               value: {{ include "rondb.mysqldPodname" . }}
             # On MySQLd Pod:
             - name: REMOTE_BACKUP_DIR
               value: /tmp/backup
             - name: LOCAL_BACKUP_DIR
               value: /home/hopsworks/schemata
-            - name: RCLONE_MOUNT_FILEPATH
-              value: &rawRCloneConf /home/hopsworks/rclone-raw.conf
             # This will be read by rclone
             - name: RCLONE_CONFIG
               value: /home/hopsworks/rclone.conf
-{{- if eq $.Values.backups.objectStorageProvider "s3" }}
-            - name: ACCESS_KEY_ID
-              valueFrom:
-                secretKeyRef:
-{{- toYaml .Values.backups.s3.keyCredentialsSecret | nindent 18 }}
-                  optional: true
-            - name: SECRET_ACCESS_KEY
-              valueFrom:
-                secretKeyRef:
-{{- toYaml .Values.backups.s3.secretCredentialsSecret | nindent 18 }}
-                  optional: true
-{{- end }}
+{{- include "rondb.backup.credentials" (dict "backupConfig" $.Values.backups "namespace" $.Release.Namespace "global" $.Values.global) | indent 12 }}
             volumeMounts:
             - name: rclone-configs
-              mountPath: *rawRCloneConf
+              mountPath: /home/hopsworks/rclone.conf
               subPath: rclone.conf
+            - name: backup-id
+              mountPath: /home/hopsworks/backup-id
           # RonDB contains a native backup protocol, which is launched through
           # the mgm client. It essentially causes every datanode to create a binary
           # backup that it stores locally.
@@ -74,15 +60,15 @@ spec:
             - /bin/bash
             - -c
             - |
-{{- include "rondb.backups.defineJobNumberEnv" $ | indent 14 }}
-              ndb_mgm --ndb-connectstring=$MGM_CONNECTSTRING -e "START BACKUP $JOB_NUMBER SNAPSHOTEND WAIT COMPLETED"
+{{- include "rondb.backups.defineBackupIdEnv" $ | indent 14 }}
+              ndb_mgm --ndb-connectstring=$MGM_CONNECTSTRING -e "START BACKUP $BACKUP_ID SNAPSHOTEND WAIT COMPLETED"
             env:
-            - name: JOB_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
             - name: MGM_CONNECTSTRING
               value: {{ include "rondb.mgmdHostname" . }}:1186
+            volumeMounts:
+            - name: backup-id
+              mountPath: /home/hopsworks/backup-id
+              readOnly: true
           containers:
           - name: upload-native-backups
             image: {{ include "image_address" (dict "image" $.Values.images.toolbox) }}
@@ -93,14 +79,15 @@ spec:
             - -c
             - |
 {{ tpl (.Files.Get "files/scripts/backups/native_upload_kubectl.sh") . | indent 14 }}
-            env:
-            - name: JOB_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
+            volumeMounts:
+            - name: backup-id
+              mountPath: /home/hopsworks/backup-id
+              readOnly: true
           volumes:
           - name: rclone-configs
             configMap:
               name: rclone-configs
+          - name: backup-id
+            emptyDir: {}
 ---
 {{- end -}}

templates/backups/restore.yaml

@@ -1,6 +1,6 @@
 # Copyright (c) 2024-2025 Hopsworks AB. All rights reserved.
 
-{{ if .Values.restoreFromBackup.backupId -}}
+{{ if include "rondb.restoreFromBackup.backupId" . -}}
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -145,7 +145,7 @@ spec:
         - name: MGMD_HOST
           value: {{ include "rondb.mgmdHostname" . }}
         - name: BACKUP_ID
-          value: {{ .Values.restoreFromBackup.backupId | int | quote }}
+          value: {{ include "rondb.restoreFromBackup.backupId" . | quote }}
         resources:
           limits:
             cpu: {{ .Values.resources.limits.cpus.restore }}
@@ -200,27 +200,14 @@ spec:
         - name: MGMD_HOST
           value: {{ include "rondb.mgmdHostname" . }}
         - name: BACKUP_ID
-          value: {{ .Values.restoreFromBackup.backupId | int | quote }}
-{{- if eq $.Values.restoreFromBackup.objectStorageProvider "s3" }}
-        - name: ACCESS_KEY_ID
-          valueFrom:
-            secretKeyRef:
-{{- toYaml $.Values.restoreFromBackup.s3.keyCredentialsSecret | nindent 14 }}
-              optional: true
-        - name: SECRET_ACCESS_KEY
-          valueFrom:
-            secretKeyRef:
-{{- toYaml $.Values.restoreFromBackup.s3.secretCredentialsSecret | nindent 14 }}
-              optional: true
-{{- end }}
-        - name: RCLONE_MOUNT_FILEPATH
-          value: {{ include "rondb.rawRCloneConf" $ }}
+          value: {{ include "rondb.restoreFromBackup.backupId" . | quote }}
+{{- include "rondb.backup.credentials" (dict "backupConfig" $.Values.restoreFromBackup "namespace" $.Release.Namespace "global" $.Values.global) | indent 8 }}
         # This will be read by rclone
         - name: RCLONE_CONFIG
           value: /home/hopsworks/rclone.conf
         volumeMounts:
         - name: rclone-configs
-          mountPath: {{ include "rondb.rawRCloneConf" $ }}
+          mountPath: /home/hopsworks/rclone.conf
           subPath: rclone.conf
         resources:
           limits:
@@ -272,7 +259,7 @@ spec:
         - name: MGMD_HOST
           value: {{ include "rondb.mgmdHostname" . }}
         - name: BACKUP_ID
-          value: {{ .Values.restoreFromBackup.backupId | int | quote }}
+          value: {{ include "rondb.restoreFromBackup.backupId" . | quote }}
         resources:
           limits:
             cpu: {{ .Values.resources.limits.cpus.restore }}
@@ -309,7 +296,7 @@ spec:
           echo "Successfully removed native backup ID $BACKUP_ID from data node Pods"
         env:
         - name: BACKUP_ID
-          value: {{ .Values.restoreFromBackup.backupId | int | quote }}
+          value: {{ include "rondb.restoreFromBackup.backupId" . | quote }}
         resources:
           limits:
             cpu: {{ .Values.resources.limits.cpus.restore }}

templates/backups/shared.yaml

@@ -1,6 +1,6 @@
 # Copyright (c) 2024-2025 Hopsworks AB. All rights reserved.
 
-{{ if or .Values.backups.enabled .Values.restoreFromBackup.backupId }}
+{{ if or (include "rondb.backups.isEnabled" .) (include "rondb.restoreFromBackup.backupId" .) }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -9,38 +9,10 @@ metadata:
 data:
     rclone.conf: |
         [{{ include "rondb.rcloneRestoreRemoteName" . }}]
-{{- if eq $.Values.restoreFromBackup.objectStorageProvider "s3" }}
-        type = s3
-        provider = {{ .Values.restoreFromBackup.s3.provider }}
-        access_key_id = REG_ACCESS_KEY_ID
-        secret_access_key = REG_SECRET_ACCESS_KEY
-        env_auth = false
-        region = {{ .Values.restoreFromBackup.s3.region }}
-{{- if .Values.restoreFromBackup.s3.serverSideEncryption }}
-        server_side_encryption = {{ .Values.restoreFromBackup.s3.serverSideEncryption }}
-{{- end }}
-        storage_class = STANDARD
-{{- if .Values.restoreFromBackup.s3.endpoint }}
-        endpoint = {{ .Values.restoreFromBackup.s3.endpoint }}
-{{- end }}
-{{- end }}
+{{ include "rondb.rcloneConfig" (dict "backupConfig" .Values.restoreFromBackup "global" .Values.global) | nindent 8}}
 
         [{{ include "rondb.rcloneBackupRemoteName" . }}]
-{{- if eq $.Values.backups.objectStorageProvider "s3" }}
-        type = s3
-        provider = {{ .Values.backups.s3.provider }}
-        access_key_id = REG_ACCESS_KEY_ID
-        secret_access_key = REG_SECRET_ACCESS_KEY
-        env_auth = false
-        region = {{ .Values.backups.s3.region }}
-{{- if .Values.backups.s3.serverSideEncryption }}
-        server_side_encryption = {{ .Values.backups.s3.serverSideEncryption }}
-{{- end }}
-        storage_class = STANDARD
-{{- if .Values.backups.s3.endpoint }}
-        endpoint = {{ .Values.backups.s3.endpoint }}
-{{- end }}
-{{- end }}
+{{ include "rondb.rcloneConfig" (dict "backupConfig" .Values.backups "global" .Values.global) | nindent 8}}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -82,4 +54,9 @@ rules:
 {{- end }}
 {{- end }}
     - {{ include "rondb.mysqldPodname" . }}
+{{- if include "rondb.backups.metadataStore.configMapName" $ }}
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["create", "get", "list", "patch"]
+{{- end }}
 {{- end }}

templates/mysqlds/_helpers.tpl

@@ -70,7 +70,7 @@
     echo "Waiting for {{ include "rondb.mysqldSetupJobName" . }} Job to have completed"
 
 {{- $waitTimeoutMinutes := .Values.timeoutsMinutes.singleSetupMySQLds }}
-{{- if .Values.restoreFromBackup.backupId }}
+{{- if include "rondb.restoreFromBackup.backupId" . }}
     {{- $waitTimeoutMinutes := (add $waitTimeoutMinutes .Values.timeoutsMinutes.restoreNativeBackup) }}
 {{- end }}
     (

templates/mysqlds/binlog_servers.yaml

@@ -67,7 +67,7 @@ spec:
 {{- if .Values.priorityClass }}
       priorityClassName: {{ .Values.priorityClass | quote }}
 {{- end  }}
-{{- if .Values.restoreFromBackup.backupId }}
+{{- if include "rondb.restoreFromBackup.backupId" . }}
       # We need to wait for the restore Job
       serviceAccountName: {{ include "rondb.serviceAccount.restoreWatcher" . }}
 {{- end }}