chore: mongodb restore failure (#2294)

Author: gnolong

addons/mongodb/Chart.yaml

@@ -4,7 +4,7 @@ description: MongoDB is a document database designed for ease of application dev
 
 type: application
 
-version: 0.9.5
+version: 0.9.6
 
 appVersion: "5.0"
 

addons/mongodb/dataprotection/common-scripts.sh

@@ -227,6 +227,8 @@ storage:
     credentials:
       access-key-id: ${S3_ACCESS_KEY}
       secret-access-key: ${S3_SECRET_KEY}
+restore:
+    numDownloadWorkers: ${PBM_RESTORE_DOWNLOAD_WORKERS:-4}
 EOF
     echo "INFO: PBM storage configuration completed."
   fi

addons/mongodb/dataprotection/rs-pbm-full-restore.sh

@@ -53,11 +53,12 @@ while [ $attempt -le $MAX_RETRIES ]; do
         continue
     else
         echo "ERROR: Failed to get backup metadata: $describe_result"
+        exit 1
     fi
 done
 set -e
 
-if [ -z "$describe_result" ]; then
+if [ -z "$describe_result" ] || echo "$describe_result" | grep -q "not found"; then
     echo "ERROR: Failed to get backup metadata after $MAX_RETRIES attempts"
     exit 1
 fi

addons/mongodb/dataprotection/shard-pbm-full-restore.sh

@@ -53,11 +53,12 @@ while [ $attempt -le $MAX_RETRIES ]; do
         continue
     else
         echo "ERROR: Failed to get backup metadata: $describe_result"
+        exit 1
     fi
 done
 set -e
 
-if [ -z "$describe_result" ]; then
+if [ -z "$describe_result" ] || echo "$describe_result" | grep -q "not found"; then
     echo "ERROR: Failed to get backup metadata after $MAX_RETRIES attempts"
     exit 1
 fi

addons/mongodb/scripts/mongodb-common.sh

@@ -5,6 +5,7 @@ function wait_restore_completion_by_cluster_cr() {
     local max_retries=$1
     local wait_interval=5
     local retries=0
+    local state=0
 
     while true; do
         cluster_json=$(kubectl get clusters.apps.kubeblocks.io "${KB_CLUSTER_NAME}" -n "${KB_NAMESPACE}" -o json 2>&1)
@@ -25,7 +26,10 @@ function wait_restore_completion_by_cluster_cr() {
             echo "INFO: No restore-from-backup annotation, do not need to restore."
             return 0
         else
-            echo "INFO: Waiting for restore completion..."
+            if [[ "$state" != "1" ]]; then
+                echo "INFO: Waiting for restore completion..."
+                state=1
+            fi
             sleep $wait_interval
             ((retries++))
             if [[ -n "$max_retries" && "$retries" -ge "$max_retries" ]]; then
@@ -53,16 +57,25 @@ function process_restore_signal() {
     local process="$1"
     local target_signal="$2"
     local pbm_backupfile=$MONGODB_ROOT/tmp/mongodb_pbm.backup
+    local last_annotation_value=""
+    local last_state="0"
     restore_signal_cm_name="$KB_CLUSTER_NAME-restore-signal" 
     restore_signal_cm_namespace="$KB_NAMESPACE"
     while true; do
         kubectl_get_result=$(kubectl get configmap $restore_signal_cm_name -n $restore_signal_cm_namespace -o json 2>&1)
         kubectl_get_exit_code=$?
         if [ "$kubectl_get_exit_code" -ne 0 ]; then
-            echo "INFO: Waiting for restore signal..."
+            if [[ "$last_state" != "1" ]]; then
+                echo "INFO: Waiting for restore signal..."
+                last_state="1"
+            fi
         else
+            last_state="0"
             annotation_value=$(echo "$kubectl_get_result" | jq -r '.metadata.labels["apps.kubeblocks.io/restore-mongodb-shard"] // empty')
-            echo "INFO: Restore signal is $annotation_value."
+            if [[ "$annotation_value" != "$last_annotation_value" ]]; then
+                echo "INFO: Restore signal is $annotation_value."
+                last_annotation_value="$annotation_value"
+            fi
             if [[ "$annotation_value" == "start" ]]; then
                 if [[ "$target_signal" == "start" ]]; then
                     echo "INFO: Restore $annotation_value signal received, starting restore..."

addons/mongodb/scripts/mongodb-shard-setup.tpl

@@ -24,7 +24,7 @@ process="mongod --bind_ip_all --port $PORT --replSet $KB_CLUSTER_COMP_NAME --con
 boot_or_enter_restore "$process"
 
 echo "INFO: Startup backup agent for restore."
-pbm-agent-entrypoint &
+pbm-agent-entrypoint >> $MONGODB_ROOT/tmp/pbm_agent_restore.log 2>&1 &
 
 echo "INFO: Start mongodb for restore."
 $process &

addons/mongodb/scripts/replicaset-setup.tpl

@@ -51,7 +51,7 @@ then
 fi
 
 echo "INFO: Startup backup agent for restore."
-pbm-agent-entrypoint &
+pbm-agent-entrypoint >> $MONGODB_ROOT/tmp/pbm_agent_restore.log 2>&1 &
 
 echo "INFO: Start mongodb for restore."
 $process &

addons/mongodb/templates/actionset-rs-pbm-physical.yaml

@@ -16,7 +16,9 @@ spec:
     - name: PBM_BACKUP_TYPE
       value: physical
     - name: PBM_COMPRESSION
-      value: zstd
+      value: s2 # for big data, use s2 compression to reduce memory usage during restoration
+    - name: PBM_RESTORE_DOWNLOAD_WORKERS
+      value: "4"
     - name: PBM_IMAGE_TAG
       value: {{ .Values.image.percona.backup.tag }}
     - name: PSM_IMAGE_TAG

addons/mongodb/templates/actionset-rs-pbm-pitr.yaml

@@ -19,7 +19,9 @@ spec:
     - name: PBM_OPLOG_SPAN_MIN_MINUTES
       value: "5"
     - name: PBM_COMPRESSION
-      value: zstd
+      value: s2 # for big data, use s2 compression to reduce memory usage during restoration
+    - name: PBM_RESTORE_DOWNLOAD_WORKERS
+      value: "4"
     - name: PBM_PURGE_INTERVAL_SECONDS
       value: "86400"
     - name: PBM_BACKUP_TYPE

addons/mongodb/templates/actionset-shard-pbm-logical.yaml

@@ -16,7 +16,9 @@ spec:
     - name: PBM_BACKUP_TYPE
       value: logical
     - name: PBM_COMPRESSION
-      value: zstd
+      value: s2 # for big data, use s2 compression to reduce memory usage during restoration
+    - name: PBM_RESTORE_DOWNLOAD_WORKERS
+      value: "4"
     - name: PBM_IMAGE_TAG
       value: {{ .Values.image.percona.backup.tag }}
     - name: PSM_IMAGE_TAG