E2E tests. Add wait flag for storage resync and update expected error message (#1156)

* PBM-1401. Add wait flag for storage resync to e2e tests
* E2E tests. Update expected error message to match actual backup failure output

Author: sandraromanchenko

e2e-tests/pkg/pbm/pbm_ctl.go

@@ -41,12 +41,12 @@ func NewCtl(ctx context.Context, host, pbmContainer string) (*Ctl, error) {
 }
 
 func (c *Ctl) PITRon() error {
-	out, err := c.RunCmd("pbm", "config", "--set", "pitr.enabled=true")
+	out, err := c.RunCmd("pbm", "config", "--set", "pitr.enabled=true", "--wait")
 	if err != nil {
 		return errors.Wrap(err, "config set pitr.enabled=true")
 	}
 
-	_, err = c.RunCmd("pbm", "config", "--set", "pitr.oplogSpanMin=1")
+	_, err = c.RunCmd("pbm", "config", "--set", "pitr.oplogSpanMin=1", "--wait")
 	if err != nil {
 		return errors.Wrap(err, "config set pitr.oplogSpanMin=1")
 	}
@@ -56,7 +56,7 @@ func (c *Ctl) PITRon() error {
 }
 
 func (c *Ctl) PITRoff() error {
-	out, err := c.RunCmd("pbm", "config", "--set", "pitr.enabled=false")
+	out, err := c.RunCmd("pbm", "config", "--set", "pitr.enabled=false", "--wait")
 	if err != nil {
 		return err
 	}
@@ -66,7 +66,7 @@ func (c *Ctl) PITRoff() error {
 }
 
 func (c *Ctl) ApplyConfig(file string) error {
-	out, err := c.RunCmd("pbm", "config", "--file", file)
+	out, err := c.RunCmd("pbm", "config", "--file", file, "--wait")
 	if err != nil {
 		return err
 	}
@@ -76,7 +76,7 @@ func (c *Ctl) ApplyConfig(file string) error {
 }
 
 func (c *Ctl) Resync() error {
-	out, err := c.RunCmd("pbm", "config", "--force-resync")
+	out, err := c.RunCmd("pbm", "config", "--force-resync", "--wait")
 	if err != nil {
 		return err
 	}

e2e-tests/pkg/tests/sharded/test_dr_restart_agents.go

@@ -11,7 +11,7 @@ import (
 
 const (
 	pbmLostAgentsErr = "some of pbm-agents were lost during the backup"
-	pbmLostShardErr  = "convergeCluster: lost shard"
+	pbmLostShardErr  = "lost shard"
 )
 
 // RestartAgents restarts agents during backup.

e2e-tests/run-new-cluster

@@ -21,7 +21,7 @@ desc 'Add test data'
 mongo_run 'for (var i = 1; i <= 50000; i++) db.getSiblingDB("test").testts.insert( { x : Date.now(), y: i } )' "rs1"
 
 desc 'Configure PBM'
-pbm_run config --file=/etc/pbm/minio.yaml
+pbm_run config --file=/etc/pbm/minio.yaml --wait
 sleep 6
 wait_noop
 pbm_run status
@@ -31,13 +31,13 @@ desc 'Backup'
 pbm_run backup
 wait_backup
 wait_noop
-pbm_run config --set pitr.enabled=true
+pbm_run config --set pitr.enabled=true --wait
 sleep 6
 mongo_run 'db.getSiblingDB("admin").createRole({ "role": "testRolePITR","privileges": [{ "resource": { "anyResource": true },"actions": [ "anyAction" ]}],"roles": []});' "rs1"
 mongo_run 'db.getSiblingDB("admin").createUser({user: "testUserPITR",pwd: "test1234","roles" : [{ "db" : "admin", "role" : "testRolePITR" }]});' "rs1"
 date
 sleep 30
-pbm_run config --set pitr.enabled=false
+pbm_run config --set pitr.enabled=false --wait
 wait_noop
 pbm_run logs -t 10
 PITR_Time=$(pbm_run status -s backups | grep -o " \- [0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}\:[0-9]\{2\}\:[0-9]\{2\}" | sed "s/ - //" | head -n 1)
@@ -62,7 +62,7 @@ mongo_run 'db.getSiblingDB("admin").createRole({ "role": "testRole2","privileges
 mongo_run 'db.getSiblingDB("admin").createUser({user: "testUser2",pwd: "test1234","roles" : [{ "db" : "admin", "role" : "testRole2" }]});' "rs1"
 
 desc 'Configure PBM'
-pbm_run config --file=/etc/pbm/minio.yaml
+pbm_run config --file=/etc/pbm/minio.yaml --wait
 pbm_run status
 sleep 6
 wait_noop