PBM-1391: Enabling PITR after physical restore causes PSMDB to crash (#1019)

* Fix inconsistent data after physical restore with PITR
After PITR physical restore, there was inconsistent data between Primary
and Secondaries nodes.
The reason was that PITR oplog and dropping collections are applied in
reverse order:
- on Primary: [PITR oplog apply] -> [dropping PBM databases]
- on each Secondary: [dropping PBM databases] -> [catch up from Primary
including oplog apply]

Not using DDL operations (drop in this case) for PBM's system
collections fixes the problem.

* Expand setupNewDB logic with all PBM's collections
That ensures that collection will not be created during PITR and by
doing that we eliminate the possible problem of having different UUIDs.

Author: boris-ilijic

cmd/pbm-agent/setup.go

@@ -110,6 +110,14 @@ func setupNewDB(ctx context.Context, conn connect.Client) error {
 		return errors.Wrap(err, "ensure pitr chunks index")
 	}
 
+	err = conn.AdminCommand(
+		ctx,
+		bson.D{{"create", defs.PITRCollection}},
+	).Err()
+	if err != nil && !strings.Contains(err.Error(), "already exists") {
+		return errors.Wrap(err, "ensure pitr collection")
+	}
+
 	_, err = conn.BcpCollection().Indexes().CreateMany(
 		ctx,
 		[]mongo.IndexModel{
@@ -124,6 +132,25 @@ func setupNewDB(ctx context.Context, conn connect.Client) error {
 			},
 		},
 	)
+	if err != nil && !strings.Contains(err.Error(), "already exists") {
+		return errors.Wrap(err, "ensure backup collection index")
+	}
+
+	err = conn.AdminCommand(
+		ctx,
+		bson.D{{"create", defs.RestoresCollection}},
+	).Err()
+	if err != nil && !strings.Contains(err.Error(), "already exists") {
+		return errors.Wrap(err, "ensure restore collection")
+	}
+
+	err = conn.AdminCommand(
+		ctx,
+		bson.D{{"create", defs.AgentsStatusCollection}},
+	).Err()
+	if err != nil && !strings.Contains(err.Error(), "already exists") {
+		return errors.Wrap(err, "ensure agent status collection")
+	}
 
-	return err
+	return nil
 }

pbm/restore/physical.go

@@ -1524,13 +1524,13 @@ func (r *PhysRestore) resetRS() error {
 			return errors.Wrap(err, "turn off pitr")
 		}
 
-		r.dropPBMCollections(ctx, c)
+		r.cleanUpPBMCollections(ctx, c)
 	}
 
 	return r.shutdown(c)
 }
 
-func (r *PhysRestore) dropPBMCollections(ctx context.Context, c *mongo.Client) {
+func (r *PhysRestore) cleanUpPBMCollections(ctx context.Context, c *mongo.Client) {
 	pbmCollections := []string{
 		defs.LockCollection,
 		defs.LogCollection,
@@ -1554,9 +1554,9 @@ func (r *PhysRestore) dropPBMCollections(ctx context.Context, c *mongo.Client) {
 			defer wg.Done()
 
 			r.log.Debug("dropping 'admin.%s'", coll)
-			err := c.Database(defs.DB).Collection(coll).Drop(ctx)
+			_, err := c.Database(defs.DB).Collection(coll).DeleteMany(ctx, bson.D{})
 			if err != nil {
-				r.log.Warning("failed to drop 'admin.%s': %v", coll, err)
+				r.log.Warning("failed to delete all from 'admin.%s': %v", coll, err)
 			}
 		}()
 	}