PBM-1401 Backup fails with context deadline exceeded error (#1150)

* hide credentials from gcs output
* add wait to file flag
* add wait implementation to set flag

Author: veceraj

cmd/pbm/common.go

@@ -44,6 +44,23 @@ func checkForAnotherOperation(ctx context.Context, pbm *sdk.Client) error {
 	return nil
 }
 
+func waitForResyncWithTimeout(ctx context.Context, pbm *sdk.Client, cid sdk.CommandID, timeout time.Duration) error {
+	if timeout > time.Second {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, timeout)
+		defer cancel()
+	}
+
+	if err := sdk.WaitForResync(ctx, pbm, cid); err != nil {
+		if errors.Is(err, context.DeadlineExceeded) {
+			err = errWaitTimeout
+		}
+
+		return errors.Wrapf(err, "waiting for resync [opid %q]", cid)
+	}
+	return nil
+}
+
 type concurrentOpError struct{ sdk.OpLock }
 
 func (e *concurrentOpError) Error() string {

cmd/pbm/config.go

@@ -77,9 +77,16 @@ func runConfig(
 			}
 		}
 		if rsnc {
-			if _, err := pbm.SyncFromStorage(ctx, false); err != nil {
+			cid, err := pbm.SyncFromStorage(ctx, false)
+			if err != nil {
 				return nil, errors.Wrap(err, "resync")
 			}
+
+			if c.wait {
+				if err := waitForResyncWithTimeout(ctx, pbm, cid, c.waitTime); err != nil {
+					return nil, err
+				}
+			}
 		}
 		return o, nil
 	case len(c.key) > 0:
@@ -101,19 +108,8 @@ func runConfig(
 			return outMsg{"Storage resync started"}, nil
 		}
 
-		if c.waitTime > time.Second {
-			var cancel context.CancelFunc
-			ctx, cancel = context.WithTimeout(ctx, c.waitTime)
-			defer cancel()
-		}
-
-		err = sdk.WaitForResync(ctx, pbm, cid)
-		if err != nil {
-			if errors.Is(err, context.DeadlineExceeded) {
-				err = errWaitTimeout
-			}
-
-			return nil, errors.Wrapf(err, "waiting for resync [opid %q]", cid)
+		if err := waitForResyncWithTimeout(ctx, pbm, cid, c.waitTime); err != nil {
+			return nil, err
 		}
 
 		return outMsg{"Storage resync finished"}, nil
@@ -144,9 +140,16 @@ func runConfig(
 
 		// resync storage only if Storage options have changed
 		if !reflect.DeepEqual(newCfg.Storage, oldCfg.Storage) {
-			if _, err := pbm.SyncFromStorage(ctx, false); err != nil {
+			cid, err := pbm.SyncFromStorage(ctx, false)
+			if err != nil {
 				return nil, errors.Wrap(err, "resync")
 			}
+
+			if c.wait {
+				if err := waitForResyncWithTimeout(ctx, pbm, cid, c.waitTime); err != nil {
+					return nil, err
+				}
+			}
 		}
 
 		return newCfg, nil

cmd/pbm/profile.go

@@ -149,19 +149,8 @@ func handleAddConfigProfile(
 		}
 
 		if opts.wait {
-			if opts.waitTime > time.Second {
-				var cancel context.CancelFunc
-				ctx, cancel = context.WithTimeout(ctx, opts.waitTime)
-				defer cancel()
-			}
-
-			err = sdk.WaitForResync(ctx, pbm, cid)
-			if err != nil {
-				if errors.Is(err, context.DeadlineExceeded) {
-					err = errWaitTimeout
-				}
-
-				return nil, errors.Wrap(err, "wait")
+			if err := waitForResyncWithTimeout(ctx, pbm, cid, opts.waitTime); err != nil {
+				return nil, err
 			}
 		}
 	}
@@ -251,19 +240,8 @@ func handleSyncConfigProfile(
 	}
 
 	if opts.wait {
-		if opts.waitTime > time.Second {
-			var cancel context.CancelFunc
-			ctx, cancel = context.WithTimeout(ctx, opts.waitTime)
-			defer cancel()
-		}
-
-		err = sdk.WaitForResync(ctx, pbm, cid)
-		if err != nil {
-			if errors.Is(err, context.DeadlineExceeded) {
-				err = errWaitTimeout
-			}
-
-			return nil, errors.Wrap(err, "wait")
+		if err := waitForResyncWithTimeout(ctx, pbm, cid, opts.waitTime); err != nil {
+			return nil, err
 		}
 	}
 

pbm/storage/gcs/gcs.go

@@ -15,7 +15,7 @@ import (
 type Config struct {
 	Bucket      string      `bson:"bucket" json:"bucket" yaml:"bucket"`
 	Prefix      string      `bson:"prefix" json:"prefix" yaml:"prefix"`
-	Credentials Credentials `bson:"credentials" json:"credentials" yaml:"credentials"`
+	Credentials Credentials `bson:"credentials" json:"-" yaml:"credentials"`
 
 	// The maximum number of bytes that the Writer will attempt to send in a single request.
 	// https://pkg.go.dev/cloud.google.com/go/storage#Writer