K8SPSMDB-1339: Validate PiTR target before starting restore

config/crd/bases/psmdb.percona.com_perconaservermongodbbackups.yaml

@@ -109,6 +109,9 @@ spec:
               lastTransition:
                 format: date-time
                 type: string
+              lastWriteAt:
+                format: date-time
+                type: string
               latestRestorableTime:
                 format: date-time
                 type: string

config/crd/bases/psmdb.percona.com_perconaservermongodbrestores.yaml

@@ -75,6 +75,9 @@ spec:
                   lastTransition:
                     format: date-time
                     type: string
+                  lastWriteAt:
+                    format: date-time
+                    type: string
                   latestRestorableTime:
                     format: date-time
                     type: string

deploy/bundle.yaml

@@ -113,6 +113,9 @@ spec:
               lastTransition:
                 format: date-time
                 type: string
+              lastWriteAt:
+                format: date-time
+                type: string
               latestRestorableTime:
                 format: date-time
                 type: string
@@ -274,6 +277,9 @@ spec:
                   lastTransition:
                     format: date-time
                     type: string
+                  lastWriteAt:
+                    format: date-time
+                    type: string
                   latestRestorableTime:
                     format: date-time
                     type: string

deploy/crd.yaml

@@ -113,6 +113,9 @@ spec:
               lastTransition:
                 format: date-time
                 type: string
+              lastWriteAt:
+                format: date-time
+                type: string
               latestRestorableTime:
                 format: date-time
                 type: string
@@ -274,6 +277,9 @@ spec:
                   lastTransition:
                     format: date-time
                     type: string
+                  lastWriteAt:
+                    format: date-time
+                    type: string
                   latestRestorableTime:
                     format: date-time
                     type: string

deploy/cw-bundle.yaml

@@ -113,6 +113,9 @@ spec:
               lastTransition:
                 format: date-time
                 type: string
+              lastWriteAt:
+                format: date-time
+                type: string
               latestRestorableTime:
                 format: date-time
                 type: string
@@ -274,6 +277,9 @@ spec:
                   lastTransition:
                     format: date-time
                     type: string
+                  lastWriteAt:
+                    format: date-time
+                    type: string
                   latestRestorableTime:
                     format: date-time
                     type: string

e2e-tests/functions

@@ -229,6 +229,10 @@ wait_backup_agent() {
 	set_debug
 }
 
+print_operator_logs() {
+	kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod)
+}
+
 wait_backup() {
 	local backup_name=$1
 	local target_state=${2:-"ready"}
@@ -243,15 +247,10 @@ wait_backup() {
 		let retry+=1
 		current_status=$(kubectl_bin get psmdb-backup $backup_name -o jsonpath='{.status.state}')
 		if [[ $retry -ge 600 || ${current_status} == 'error' ]]; then
-			kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) \
-				| grep -v 'level=info' \
-				| grep -v 'level=debug' \
-				| grep -v 'Getting tasks for pod' \
-				| grep -v 'Getting pods from source' \
-				| tail -200
+			print_operator_logs | tail -n 200
 			kubectl_bin get psmdb-backup
+			kubectl_bin describe psmdb-backup/${backup_name}
 			echo "Backup object psmdb-backup/${backup_name} is in ${current_status} state."
-			echo something went wrong with operator or kubernetes cluster
 			exit 1
 		fi
 	done
@@ -356,7 +355,7 @@ get_mongod_pods() {
 	kubectl_bin get pod \
 		--no-headers \
 		-l app.kubernetes.io/instance=${cluster} \
-		-l app.kubernetes.io/component=mongod
+		-l app.kubernetes.io/component=mongod | awk '{print $1}'
 
 }
 
@@ -366,7 +365,7 @@ collect_physical_restore_logs() {
 
 	for pod in $(get_mongod_pods ${cluster}); do
 		desc "pbm-agent logs from ${pod}"
-		kubectl_bin exec -it ${pod} -- cat /tmp/pbm-agent.log || true
+		kubectl_bin exec ${pod} -- cat /tmp/pbm-agent.log || echo "failed to get /tmp/pbm-agent.log from ${pod}"
 	done
 }
 

e2e-tests/pitr/conf/some-name-rs0.yml

@@ -22,7 +22,6 @@ spec:
       enabled: true
       oplogSpanMin: 2
       oplogOnly: false
-      oplogSpanMin: 10
       compressionType: gzip
       compressionLevel: 6
   replsets:

e2e-tests/version-service/conf/crd.yaml

@@ -113,6 +113,9 @@ spec:
               lastTransition:
                 format: date-time
                 type: string
+              lastWriteAt:
+                format: date-time
+                type: string
               latestRestorableTime:
                 format: date-time
                 type: string
@@ -274,6 +277,9 @@ spec:
                   lastTransition:
                     format: date-time
                     type: string
+                  lastWriteAt:
+                    format: date-time
+                    type: string
                   latestRestorableTime:
                     format: date-time
                     type: string

pkg/apis/psmdb/v1/perconaservermongodbbackup_types.go

@@ -34,25 +34,27 @@ const (
 
 // PerconaServerMongoDBBackupStatus defines the observed state of PerconaServerMongoDBBackup
 type PerconaServerMongoDBBackupStatus struct {
-	Type           defs.BackupType              `json:"type,omitempty"`
-	State          BackupState                  `json:"state,omitempty"`
-	StartAt        *metav1.Time                 `json:"start,omitempty"`
-	CompletedAt    *metav1.Time                 `json:"completed,omitempty"`
-	LastTransition *metav1.Time                 `json:"lastTransition,omitempty"`
-	Destination    string                       `json:"destination,omitempty"`
-	StorageName    string                       `json:"storageName,omitempty"`
-	S3             *BackupStorageS3Spec         `json:"s3,omitempty"`
-	Azure          *BackupStorageAzureSpec      `json:"azure,omitempty"`
-	Filesystem     *BackupStorageFilesystemSpec `json:"filesystem,omitempty"`
-	ReplsetNames   []string                     `json:"replsetNames,omitempty"`
-	PBMname        string                       `json:"pbmName,omitempty"`
-	Size           string                       `json:"size,omitempty"`
+	Type         defs.BackupType              `json:"type,omitempty"`
+	State        BackupState                  `json:"state,omitempty"`
+	Destination  string                       `json:"destination,omitempty"`
+	StorageName  string                       `json:"storageName,omitempty"`
+	S3           *BackupStorageS3Spec         `json:"s3,omitempty"`
+	Azure        *BackupStorageAzureSpec      `json:"azure,omitempty"`
+	Filesystem   *BackupStorageFilesystemSpec `json:"filesystem,omitempty"`
+	ReplsetNames []string                     `json:"replsetNames,omitempty"`
+	PBMname      string                       `json:"pbmName,omitempty"`
+	Size         string                       `json:"size,omitempty"`
 
 	// Deprecated: Use PBMPods instead
-	PBMPod               string            `json:"pbmPod,omitempty"`
-	PBMPods              map[string]string `json:"pbmPods,omitempty"`
-	Error                string            `json:"error,omitempty"`
-	LatestRestorableTime *metav1.Time      `json:"latestRestorableTime,omitempty"`
+	PBMPod  string            `json:"pbmPod,omitempty"`
+	PBMPods map[string]string `json:"pbmPods,omitempty"`
+	Error   string            `json:"error,omitempty"`
+
+	StartAt              *metav1.Time `json:"start,omitempty"`
+	CompletedAt          *metav1.Time `json:"completed,omitempty"`
+	LastWriteAt          *metav1.Time `json:"lastWriteAt,omitempty"`
+	LastTransition       *metav1.Time `json:"lastTransition,omitempty"`
+	LatestRestorableTime *metav1.Time `json:"latestRestorableTime,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/controller/perconaservermongodbbackup/backup.go

@@ -198,6 +198,9 @@ func (b *Backup) Status(ctx context.Context, cr *api.PerconaServerMongoDBBackup)
 		status.CompletedAt = &metav1.Time{
 			Time: time.Unix(meta.LastTransitionTS, 0),
 		}
+		status.LastWriteAt = &metav1.Time{
+			Time: time.Unix(int64(meta.LastWriteTS.T), 0),
+		}
 	case defs.StatusStarting:
 		passed := time.Now().UTC().Sub(time.Unix(meta.StartTS, 0))
 		if passed >= pbmStartingDeadline {

pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go

@@ -188,7 +188,7 @@ func (r *ReconcilePerconaServerMongoDBRestore) Reconcile(ctx context.Context, re
 	}
 
 	if cr.Status.State == psmdbv1.RestoreStateNew {
-		err := r.validate(ctx, cr, cluster)
+		err = r.validate(ctx, cr, cluster)
 		if err != nil {
 			if errors.Is(err, errWaitingPBM) {
 				err = nil
@@ -209,7 +209,7 @@ func (r *ReconcilePerconaServerMongoDBRestore) Reconcile(ctx context.Context, re
 				return rr, nil
 			}
 
-			return rr, errors.Wrap(err, "failed to validate restore")
+			return reconcile.Result{}, errors.Wrap(err, "failed to validate restore")
 		}
 
 		if cluster.Spec.Sharding.Enabled {

pkg/controller/perconaservermongodbrestore/validate.go

@@ -49,5 +49,32 @@ func (r *ReconcilePerconaServerMongoDBRestore) validate(ctx context.Context, cr
 		return errors.Wrap(err, "failed to validate backup")
 	}
 
+	pitr := cr.Spec.PITR
+	if pitr == nil {
+		return nil
+	}
+
+	switch {
+	case pitr.Type == psmdbv1.PITRestoreTypeDate && pitr.Date != nil:
+		if bcp.Status.LastWriteAt != nil {
+			if pitr.Date.Equal(bcp.Status.LastWriteAt) {
+				return errors.New("backup's last write is equal to target time")
+			}
+			if pitr.Date.Before(bcp.Status.LastWriteAt) {
+				return errors.New("backup's last write is later than target time")
+			}
+		}
+
+		ts := pitr.Date.Unix()
+		if _, err := pbmc.GetPITRChunkContains(ctx, ts); err != nil {
+			return err
+		}
+	case pitr.Type == psmdbv1.PITRestoreTypeLatest:
+		_, err := pbmc.GetLatestTimelinePITR(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }