PBM-1447. Unset LD_PRELOAD on physical restore to prevent PSMDB 8.0 from getting stuck (#1064)

Author: sandraromanchenko

e2e-tests/cmd/pbm-test/run_physical.go

@@ -4,8 +4,6 @@ import (
 	"context"
 	"math/rand"
 
-	"golang.org/x/mod/semver"
-
 	"github.com/percona/percona-backup-mongodb/e2e-tests/pkg/tests/sharded"
 	"github.com/percona/percona-backup-mongodb/pbm/defs"
 )
@@ -60,11 +58,8 @@ func runPhysical(t *sharded.Cluster, typ testTyp) {
 			t.DistributedTrxPhysical)
 	}
 
-	// Skip test for 8.0 until PBM-1447 is fixed
-	if semver.Compare(cVersion, "v8.0") < 0 {
-		runTest("Clock Skew Tests",
-			func() { t.ClockSkew(defs.PhysicalBackup, cVersion) })
-	}
+	runTest("Clock Skew Tests",
+		func() { t.ClockSkew(defs.PhysicalBackup, cVersion) })
 
 	flushStore(t)
 }

e2e-tests/pkg/pbm/clock_skew.go

@@ -3,6 +3,7 @@ package pbm
 import (
 	"context"
 	"log"
+	"strings"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
@@ -13,7 +14,9 @@ import (
 )
 
 func ClockSkew(rsName, ts, dockerHost string) error {
-	log.Printf("== Skew the clock for %s on the replicaset %s ", ts, rsName)
+	if ts != "0" {
+		log.Printf("==Skew the clock for %s on the replicaset %s ", ts, rsName)
+	}
 
 	cn, err := client.NewClientWithOpts(client.WithHost(dockerHost))
 	if err != nil {
@@ -36,18 +39,41 @@ func ClockSkew(rsName, ts, dockerHost string) error {
 			return errors.Wrapf(err, "ContainerInspect for %s", c.ID)
 		}
 
+		envs := append([]string{}, containerOld.Config.Env...)
+		if ts == "0" {
+			ldPreloadDefined := false
+			for _, env := range envs {
+				if strings.HasPrefix(env, "LD_PRELOAD") {
+					ldPreloadDefined = true
+					break
+				}
+			}
+
+			if !ldPreloadDefined {
+				log.Printf("Variable LD_PRELOAD isn't defined, skipping container restart for %s\n", containerOld.Name)
+				continue
+			}
+
+			var filteredEnvs []string
+			for _, env := range envs {
+				if !strings.HasPrefix(env, "LD_PRELOAD") {
+					filteredEnvs = append(filteredEnvs, env)
+				}
+			}
+			envs = filteredEnvs
+		} else {
+			envs = append(envs,
+				`LD_PRELOAD=/lib64/faketime/libfaketime.so.1`,
+				`FAKETIME=`+ts,
+			)
+		}
+
 		log.Printf("Removing container %s/%s\n", containerOld.ID, containerOld.Name)
 		err = cn.ContainerRemove(context.Background(), c.ID, container.RemoveOptions{Force: true})
 		if err != nil {
 			return errors.Wrapf(err, "remove container %s", c.ID)
 		}
 
-		envs := append([]string{}, containerOld.Config.Env...)
-		envs = append(envs,
-			`LD_PRELOAD=/lib64/faketime/libfaketime.so.1`,
-			`FAKETIME=`+ts,
-		)
-
 		log.Printf("Creating container %s/%s with the clock skew %s\n", containerOld.ID, containerOld.Name, ts)
 		containerNew, err := cn.ContainerCreate(context.Background(), &container.Config{
 			Image:  containerOld.Image,

e2e-tests/pkg/tests/sharded/cluster.go

@@ -145,6 +145,14 @@ func (c *Cluster) PhysicalRestore(ctx context.Context, bcpName string) {
 }
 
 func (c *Cluster) PhysicalRestoreWithParams(ctx context.Context, bcpName string, options []string) {
+	stdlog.Println("reset ENV variables")
+	for name := range c.shards {
+		err := pbmt.ClockSkew(name, "0", c.cfg.DockerURI)
+		if err != nil {
+			stdlog.Fatalln("reset ENV variables:", err)
+		}
+	}
+
 	stdlog.Println("restoring the backup")
 	name, err := c.pbm.Restore(bcpName, options)
 	if err != nil {

e2e-tests/pkg/tests/sharded/test_clock_skew.go

@@ -2,6 +2,7 @@ package sharded
 
 import (
 	"log"
+	"math/rand"
 
 	pbmt "github.com/percona/percona-backup-mongodb/e2e-tests/pkg/pbm"
 	"github.com/percona/percona-backup-mongodb/pbm/defs"
@@ -11,16 +12,14 @@ func (c *Cluster) ClockSkew(typ defs.BackupType, mongoVersion string) {
 	timeShifts := []string{
 		"+90m", "-195m", "+2d", "-7h", "+11m", "+42d", "-13h",
 	}
-	rsNames := []string{"cfg"}
+	rsNames := []string{}
 	for s := range c.shards {
 		rsNames = append(rsNames, s)
 	}
 
-	for k, rs := range rsNames {
-		if k >= len(timeShifts) {
-			k %= len(timeShifts)
-		}
-		shift := timeShifts[k]
+	for _, rs := range rsNames {
+		randomIndex := rand.Intn(len(timeShifts))
+		shift := timeShifts[randomIndex]
 
 		err := pbmt.ClockSkew(rs, shift, c.cfg.DockerURI)
 		if err != nil {