diff --git a/addons/mongodb/Chart.yaml b/addons/mongodb/Chart.yaml index 15bc7611f..6beb19978 100644 --- a/addons/mongodb/Chart.yaml +++ b/addons/mongodb/Chart.yaml @@ -4,7 +4,7 @@ description: MongoDB is a document database designed for ease of application dev type: application -version: 0.9.5 +version: 0.9.6 appVersion: "5.0" diff --git a/addons/mongodb/dataprotection/common-scripts.sh b/addons/mongodb/dataprotection/common-scripts.sh index 7696ccfd7..9f08873f1 100644 --- a/addons/mongodb/dataprotection/common-scripts.sh +++ b/addons/mongodb/dataprotection/common-scripts.sh @@ -227,6 +227,8 @@ storage: credentials: access-key-id: ${S3_ACCESS_KEY} secret-access-key: ${S3_SECRET_KEY} +restore: + numDownloadWorkers: ${PBM_RESTORE_DOWNLOAD_WORKERS:-4} EOF echo "INFO: PBM storage configuration completed." fi diff --git a/addons/mongodb/dataprotection/rs-pbm-full-restore.sh b/addons/mongodb/dataprotection/rs-pbm-full-restore.sh index 30b7066da..b88526fab 100644 --- a/addons/mongodb/dataprotection/rs-pbm-full-restore.sh +++ b/addons/mongodb/dataprotection/rs-pbm-full-restore.sh @@ -53,11 +53,12 @@ while [ $attempt -le $MAX_RETRIES ]; do continue else echo "ERROR: Failed to get backup metadata: $describe_result" + exit 1 fi done set -e -if [ -z "$describe_result" ]; then +if [ -z "$describe_result" ] || echo "$describe_result" | grep -q "not found"; then echo "ERROR: Failed to get backup metadata after $MAX_RETRIES attempts" exit 1 fi diff --git a/addons/mongodb/dataprotection/shard-pbm-full-restore.sh b/addons/mongodb/dataprotection/shard-pbm-full-restore.sh index b5c59f16e..d9bcaa95d 100644 --- a/addons/mongodb/dataprotection/shard-pbm-full-restore.sh +++ b/addons/mongodb/dataprotection/shard-pbm-full-restore.sh @@ -53,11 +53,12 @@ while [ $attempt -le $MAX_RETRIES ]; do continue else echo "ERROR: Failed to get backup metadata: $describe_result" + exit 1 fi done set -e -if [ -z "$describe_result" ]; then +if [ -z "$describe_result" ] || echo "$describe_result" | grep -q "not found"; then echo "ERROR: Failed to get backup metadata after $MAX_RETRIES attempts" exit 1 fi diff --git a/addons/mongodb/scripts/mongodb-common.sh b/addons/mongodb/scripts/mongodb-common.sh index 3a4e4f7d0..6cd1a8aa1 100644 --- a/addons/mongodb/scripts/mongodb-common.sh +++ b/addons/mongodb/scripts/mongodb-common.sh @@ -5,6 +5,7 @@ function wait_restore_completion_by_cluster_cr() { local max_retries=$1 local wait_interval=5 local retries=0 + local state=0 while true; do cluster_json=$(kubectl get clusters.apps.kubeblocks.io "${KB_CLUSTER_NAME}" -n "${KB_NAMESPACE}" -o json 2>&1) @@ -25,7 +26,10 @@ function wait_restore_completion_by_cluster_cr() { echo "INFO: No restore-from-backup annotation, do not need to restore." return 0 else - echo "INFO: Waiting for restore completion..." + if [[ "$state" != "1" ]]; then + echo "INFO: Waiting for restore completion..." + state=1 + fi sleep $wait_interval ((retries++)) if [[ -n "$max_retries" && "$retries" -ge "$max_retries" ]]; then @@ -53,16 +57,25 @@ function process_restore_signal() { local process="$1" local target_signal="$2" local pbm_backupfile=$MONGODB_ROOT/tmp/mongodb_pbm.backup + local last_annotation_value="" + local last_state="0" restore_signal_cm_name="$KB_CLUSTER_NAME-restore-signal" restore_signal_cm_namespace="$KB_NAMESPACE" while true; do kubectl_get_result=$(kubectl get configmap $restore_signal_cm_name -n $restore_signal_cm_namespace -o json 2>&1) kubectl_get_exit_code=$? if [ "$kubectl_get_exit_code" -ne 0 ]; then - echo "INFO: Waiting for restore signal..." + if [[ "$last_state" != "1" ]]; then + echo "INFO: Waiting for restore signal..." + last_state="1" + fi else + last_state="0" annotation_value=$(echo "$kubectl_get_result" | jq -r '.metadata.labels["apps.kubeblocks.io/restore-mongodb-shard"] // empty') - echo "INFO: Restore signal is $annotation_value." + if [[ "$annotation_value" != "$last_annotation_value" ]]; then + echo "INFO: Restore signal is $annotation_value." + last_annotation_value="$annotation_value" + fi if [[ "$annotation_value" == "start" ]]; then if [[ "$target_signal" == "start" ]]; then echo "INFO: Restore $annotation_value signal received, starting restore..." diff --git a/addons/mongodb/scripts/mongodb-shard-setup.tpl b/addons/mongodb/scripts/mongodb-shard-setup.tpl index 744201e21..2b91bb87c 100644 --- a/addons/mongodb/scripts/mongodb-shard-setup.tpl +++ b/addons/mongodb/scripts/mongodb-shard-setup.tpl @@ -24,7 +24,7 @@ process="mongod --bind_ip_all --port $PORT --replSet $KB_CLUSTER_COMP_NAME --con boot_or_enter_restore "$process" echo "INFO: Startup backup agent for restore." -pbm-agent-entrypoint & +pbm-agent-entrypoint >> $MONGODB_ROOT/tmp/pbm_agent_restore.log 2>&1 & echo "INFO: Start mongodb for restore." $process & diff --git a/addons/mongodb/scripts/replicaset-setup.tpl b/addons/mongodb/scripts/replicaset-setup.tpl index 561170dcd..1b351aee7 100644 --- a/addons/mongodb/scripts/replicaset-setup.tpl +++ b/addons/mongodb/scripts/replicaset-setup.tpl @@ -51,7 +51,7 @@ then fi echo "INFO: Startup backup agent for restore." -pbm-agent-entrypoint & +pbm-agent-entrypoint >> $MONGODB_ROOT/tmp/pbm_agent_restore.log 2>&1 & echo "INFO: Start mongodb for restore." $process & diff --git a/addons/mongodb/templates/actionset-rs-pbm-physical.yaml b/addons/mongodb/templates/actionset-rs-pbm-physical.yaml index 3106cedf3..5d7b56a35 100644 --- a/addons/mongodb/templates/actionset-rs-pbm-physical.yaml +++ b/addons/mongodb/templates/actionset-rs-pbm-physical.yaml @@ -16,7 +16,9 @@ spec: - name: PBM_BACKUP_TYPE value: physical - name: PBM_COMPRESSION - value: zstd + value: s2 # for big data, use s2 compression to reduce memory usage during restoration + - name: PBM_RESTORE_DOWNLOAD_WORKERS + value: "4" - name: PBM_IMAGE_TAG value: {{ .Values.image.percona.backup.tag }} - name: PSM_IMAGE_TAG diff --git a/addons/mongodb/templates/actionset-rs-pbm-pitr.yaml b/addons/mongodb/templates/actionset-rs-pbm-pitr.yaml index 4fa0f212e..7f7ae99dc 100644 --- a/addons/mongodb/templates/actionset-rs-pbm-pitr.yaml +++ b/addons/mongodb/templates/actionset-rs-pbm-pitr.yaml @@ -19,7 +19,9 @@ spec: - name: PBM_OPLOG_SPAN_MIN_MINUTES value: "5" - name: PBM_COMPRESSION - value: zstd + value: s2 # for big data, use s2 compression to reduce memory usage during restoration + - name: PBM_RESTORE_DOWNLOAD_WORKERS + value: "4" - name: PBM_PURGE_INTERVAL_SECONDS value: "86400" - name: PBM_BACKUP_TYPE diff --git a/addons/mongodb/templates/actionset-shard-pbm-logical.yaml b/addons/mongodb/templates/actionset-shard-pbm-logical.yaml index b6fdee2da..970b015ca 100644 --- a/addons/mongodb/templates/actionset-shard-pbm-logical.yaml +++ b/addons/mongodb/templates/actionset-shard-pbm-logical.yaml @@ -16,7 +16,9 @@ spec: - name: PBM_BACKUP_TYPE value: logical - name: PBM_COMPRESSION - value: zstd + value: s2 # for big data, use s2 compression to reduce memory usage during restoration + - name: PBM_RESTORE_DOWNLOAD_WORKERS + value: "4" - name: PBM_IMAGE_TAG value: {{ .Values.image.percona.backup.tag }} - name: PSM_IMAGE_TAG diff --git a/addons/mongodb/templates/actionset-shard-pbm-physical.yaml b/addons/mongodb/templates/actionset-shard-pbm-physical.yaml index 34f2f39b5..923a86cc7 100644 --- a/addons/mongodb/templates/actionset-shard-pbm-physical.yaml +++ b/addons/mongodb/templates/actionset-shard-pbm-physical.yaml @@ -16,7 +16,9 @@ spec: - name: PBM_BACKUP_TYPE value: physical - name: PBM_COMPRESSION - value: zstd + value: s2 # for big data, use s2 compression to reduce memory usage during restoration + - name: PBM_RESTORE_DOWNLOAD_WORKERS + value: "4" - name: PBM_IMAGE_TAG value: {{ .Values.image.percona.backup.tag }} - name: PSM_IMAGE_TAG diff --git a/addons/mongodb/templates/actionset-shard-pbm-pitr.yaml b/addons/mongodb/templates/actionset-shard-pbm-pitr.yaml index 3b32ae0df..ff9e951ed 100644 --- a/addons/mongodb/templates/actionset-shard-pbm-pitr.yaml +++ b/addons/mongodb/templates/actionset-shard-pbm-pitr.yaml @@ -19,7 +19,9 @@ spec: - name: PBM_OPLOG_SPAN_MIN_MINUTES value: "5" - name: PBM_COMPRESSION - value: zstd + value: s2 # for big data, use s2 compression to reduce memory usage during restoration + - name: PBM_RESTORE_DOWNLOAD_WORKERS + value: "4" - name: PBM_PURGE_INTERVAL_SECONDS value: "86400" - name: PBM_BACKUP_TYPE diff --git a/addons/mongodb/templates/backuppolicytemplate.yaml b/addons/mongodb/templates/backuppolicytemplate.yaml index 56ad436f9..391855c35 100644 --- a/addons/mongodb/templates/backuppolicytemplate.yaml +++ b/addons/mongodb/templates/backuppolicytemplate.yaml @@ -103,8 +103,10 @@ spec: - key: PBM_IMAGE_TAG valueFrom: serviceVersion: - - names: ["6.0.21", "7.0.18", "8.0.8"] - mappingValue: "2.9.1" + - names: ["7.0.18", "8.0.8"] + mappingValue: "2.12.0" + - names: ["6.0.21"] + mappingValue: "2.10.0" - names: ["5.0.29", "4.4.29", "4.2.25", "4.0.28"] mappingValue: "2.5.0-multi" - key: PSM_IMAGE_TAG @@ -135,8 +137,10 @@ spec: - key: PBM_IMAGE_TAG valueFrom: serviceVersion: - - names: ["6.0.21", "7.0.18", "8.0.8"] - mappingValue: "2.9.1" + - names: ["7.0.18", "8.0.8"] + mappingValue: "2.12.0" + - names: ["6.0.21"] + mappingValue: "2.10.0" - names: ["5.0.29", "4.4.29", "4.2.25", "4.0.28"] mappingValue: "2.5.0-multi" - key: PSM_IMAGE_TAG @@ -203,8 +207,10 @@ spec: - key: PBM_IMAGE_TAG valueFrom: serviceVersion: - - names: ["6.0.21", "7.0.18", "8.0.8"] - mappingValue: "2.9.1" + - names: ["7.0.18", "8.0.8"] + mappingValue: "2.12.0" + - names: ["6.0.21"] + mappingValue: "2.10.0" - names: ["5.0.29", "4.4.29", "4.2.25", "4.0.28"] mappingValue: "2.5.0-multi" - key: PSM_IMAGE_TAG @@ -235,8 +241,10 @@ spec: - key: PBM_IMAGE_TAG valueFrom: serviceVersion: - - names: ["6.0.21", "7.0.18", "8.0.8"] - mappingValue: "2.9.1" + - names: ["7.0.18", "8.0.8"] + mappingValue: "2.12.0" + - names: ["6.0.21"] + mappingValue: "2.10.0" - names: ["5.0.29", "4.4.29", "4.2.25", "4.0.28"] mappingValue: "2.5.0-multi" - key: PSM_IMAGE_TAG @@ -267,8 +275,10 @@ spec: - key: PBM_IMAGE_TAG valueFrom: serviceVersion: - - names: ["6.0.21", "7.0.18", "8.0.8"] - mappingValue: "2.9.1" + - names: ["7.0.18", "8.0.8"] + mappingValue: "2.12.0" + - names: ["6.0.21"] + mappingValue: "2.10.0" - names: ["5.0.29", "4.4.29", "4.2.25", "4.0.28"] mappingValue: "2.5.0-multi" - key: PSM_IMAGE_TAG @@ -317,4 +327,4 @@ spec: - backupMethod: archive-oplog enabled: false cronExpression: "*/5 * * * *" - retentionPeriod: 8d \ No newline at end of file + retentionPeriod: 8d diff --git a/addons/mongodb/templates/cmpv-shard.yaml b/addons/mongodb/templates/cmpv-shard.yaml index 6b0fdb767..4b43b285c 100644 --- a/addons/mongodb/templates/cmpv-shard.yaml +++ b/addons/mongodb/templates/cmpv-shard.yaml @@ -32,30 +32,30 @@ spec: serviceVersion: 8.0.8 images: mongodb: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.repository }}:8.0.8 - mongodb-backup-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.9.1 + mongodb-backup-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.12.0 init-syncer: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.syncer.repository }}:{{ .Values.image.syncer.tag }} init-kubectl: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.kubectl.repository }}:{{ .Values.image.kubectl.tag }} - init-pbm-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.9.1 + init-pbm-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.12.0 exporter: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.exporter.repository }}:{{ .Values.image.exporter.tag }} - name: 7.0.18 changes: serviceVersion: 7.0.18 images: mongodb: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.repository }}:7.0.18 - mongodb-backup-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.9.1 + mongodb-backup-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.12.0 init-syncer: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.syncer.repository }}:{{ .Values.image.syncer.tag }} init-kubectl: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.kubectl.repository }}:{{ .Values.image.kubectl.tag }} - init-pbm-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.9.1 + init-pbm-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.12.0 exporter: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.exporter.repository }}:{{ .Values.image.exporter.tag }} - name: 6.0.21 changes: serviceVersion: 6.0.21 images: mongodb: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.repository }}:6.0.21 - mongodb-backup-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.9.1 + mongodb-backup-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.10.0 init-syncer: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.syncer.repository }}:{{ .Values.image.syncer.tag }} init-kubectl: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.kubectl.repository }}:{{ .Values.image.kubectl.tag }} - init-pbm-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.9.1 + init-pbm-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.10.0 exporter: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.exporter.repository }}:{{ .Values.image.exporter.tag }} - name: 5.0.29 changes: diff --git a/addons/mongodb/templates/componentversion.yaml b/addons/mongodb/templates/componentversion.yaml index f87d0cad7..bf47ef781 100644 --- a/addons/mongodb/templates/componentversion.yaml +++ b/addons/mongodb/templates/componentversion.yaml @@ -22,30 +22,30 @@ spec: serviceVersion: 8.0.8 images: mongodb: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:8.0.8 - mongodb-backup-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.9.1 + mongodb-backup-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.12.0 init-syncer: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.syncer.repository }}:{{ .Values.image.syncer.tag }} init-kubectl: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.kubectl.repository }}:{{ .Values.image.kubectl.tag }} - init-pbm-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.9.1 + init-pbm-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.12.0 exporter: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.exporter.repository }}:{{ .Values.image.exporter.tag }} - name: 7.0.18 changes: serviceVersion: 7.0.18 images: mongodb: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:7.0.18 - mongodb-backup-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.9.1 + mongodb-backup-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.12.0 init-syncer: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.syncer.repository }}:{{ .Values.image.syncer.tag }} init-kubectl: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.kubectl.repository }}:{{ .Values.image.kubectl.tag }} - init-pbm-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.9.1 + init-pbm-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.12.0 exporter: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.exporter.repository }}:{{ .Values.image.exporter.tag }} - name: 6.0.21 changes: serviceVersion: 6.0.21 images: mongodb: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:6.0.21 - mongodb-backup-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.9.1 + mongodb-backup-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.10.0 init-syncer: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.syncer.repository }}:{{ .Values.image.syncer.tag }} init-kubectl: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.kubectl.repository }}:{{ .Values.image.kubectl.tag }} - init-pbm-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.9.1 + init-pbm-agent: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:2.10.0 exporter: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.exporter.repository }}:{{ .Values.image.exporter.tag }} - name: 5.0.29 changes: diff --git a/addons/mongodb/values.yaml b/addons/mongodb/values.yaml index c77e3656c..c53539260 100644 --- a/addons/mongodb/values.yaml +++ b/addons/mongodb/values.yaml @@ -10,7 +10,7 @@ image: tag: 5.0.29 # 6.0.3-debian-11-r0 syncer: repository: apecloud/syncer - tag: 0.6.6 + tag: 0.6.7 percona: repository: apecloud/percona-server-mongodb tag: 8.0.8