Merge pull request #1136 from percona/PBM-1514-Improve-resync-performance-updates

PBM-1514 improve resync performance - update default behavior

Author: veceraj

cmd/pbm-agent/resync.go

@@ -71,7 +71,7 @@ func (a *Agent) Resync(ctx context.Context, cmd *ctrl.ResyncCmd, opid ctrl.OPID,
 	} else if cmd.Name != "" {
 		err = a.handleSyncProfile(ctx, cmd.Name, cmd.Clear)
 	} else {
-		err = a.handleSyncMainStorage(ctx, cmd.SkipRestores)
+		err = a.handleSyncMainStorage(ctx, cmd.IncludeRestores)
 	}
 	if err != nil {
 		l.Error(err.Error())
@@ -134,13 +134,13 @@ func (a *Agent) helpSyncProfileBackups(ctx context.Context, profile *config.Conf
 	return errors.Wrapf(err, "sync backup list for %q", profile.Name)
 }
 
-func (a *Agent) handleSyncMainStorage(ctx context.Context, skipRestores bool) error {
+func (a *Agent) handleSyncMainStorage(ctx context.Context, includeRestores bool) error {
 	cfg, err := config.GetConfig(ctx, a.leadConn)
 	if err != nil {
 		return errors.Wrap(err, "get config")
 	}
 
-	err = resync.Resync(ctx, a.leadConn, &cfg.Storage, a.brief.Me, skipRestores)
+	err = resync.Resync(ctx, a.leadConn, &cfg.Storage, a.brief.Me, includeRestores)
 	if err != nil {
 		return errors.Wrap(err, "resync")
 	}

cmd/pbm/config.go

@@ -18,14 +18,14 @@ import (
 )
 
 type configOpts struct {
-	rsync        bool
-	skipRestores bool
-	wait         bool
-	waitTime     time.Duration
-	list         bool
-	file         string
-	set          map[string]string
-	key          string
+	rsync           bool
+	includeRestores bool
+	wait            bool
+	waitTime        time.Duration
+	list            bool
+	file            string
+	set             map[string]string
+	key             string
 }
 
 type confKV struct {
@@ -92,7 +92,7 @@ func runConfig(
 		}
 		return confKV{c.key, fmt.Sprint(k)}, nil
 	case c.rsync:
-		cid, err := pbm.SyncFromStorage(ctx, c.skipRestores)
+		cid, err := pbm.SyncFromStorage(ctx, c.includeRestores)
 		if err != nil {
 			return nil, errors.Wrap(err, "resync")
 		}

cmd/pbm/main.go

@@ -368,8 +368,8 @@ func (app *pbmApp) buildConfigCmd() *cobra.Command {
 		Use:   "config [key]",
 		Short: "Set, change or list the config",
 		RunE: app.wrapRunE(func(cmd *cobra.Command, args []string) (fmt.Stringer, error) {
-			if cfg.skipRestores && !cfg.rsync {
-				return nil, errors.New("--skip-restores cannot be used without --force-resync")
+			if cfg.includeRestores && !cfg.rsync {
+				return nil, errors.New("--include-restores cannot be used without --force-resync")
 			}
 
 			if len(args) == 1 {
@@ -381,7 +381,7 @@ func (app *pbmApp) buildConfigCmd() *cobra.Command {
 
 	configCmd.Flags().BoolVar(&cfg.rsync, "force-resync", false, "Resync backup list with the current store")
 	configCmd.Flags().BoolVar(
-		&cfg.skipRestores, "skip-restores", false, "Skip physical restore metadata during force-resync",
+		&cfg.includeRestores, "include-restores", false, "Include physical restore metadata during force-resync",
 	)
 	configCmd.Flags().BoolVar(&cfg.list, "list", false, "List current settings")
 	configCmd.Flags().StringVar(&cfg.file, "file", "", "Upload config from YAML file")

pbm/ctrl/cmd.go

@@ -121,10 +121,10 @@ type ProfileCmd struct {
 }
 
 type ResyncCmd struct {
-	Name         string `bson:"name,omitempty"`
-	All          bool   `bson:"all,omitempty"`
-	Clear        bool   `bson:"clear,omitempty"`
-	SkipRestores bool   `bson:"skipRestores,omitempty"`
+	Name            string `bson:"name,omitempty"`
+	All             bool   `bson:"all,omitempty"`
+	Clear           bool   `bson:"clear,omitempty"`
+	IncludeRestores bool   `bson:"includeRestores,omitempty"`
 }
 
 type BackupCmd struct {

pbm/resync/rsync.go

@@ -31,7 +31,7 @@ func Resync(
 	conn connect.Client,
 	cfg *config.StorageConf,
 	node string,
-	skipRestores bool,
+	includeRestores bool,
 ) error {
 	l := log.LogEventFromContext(ctx)
 
@@ -68,7 +68,7 @@ func Resync(
 		l.Error("failed sync oplog range: %v", err)
 	}
 
-	err = resyncPhysicalRestores(ctx, conn, stg, skipRestores)
+	err = resyncPhysicalRestores(ctx, conn, stg, includeRestores)
 	if err != nil {
 		l.Error("failed sync physical restore metadata: %v", err)
 	}
@@ -250,7 +250,7 @@ func resyncPhysicalRestores(
 	ctx context.Context,
 	conn connect.Client,
 	stg storage.Storage,
-	skipRestores bool,
+	includeRestores bool,
 ) error {
 	_, err := conn.RestoresCollection().DeleteMany(ctx, bson.D{})
 	if err != nil {
@@ -269,7 +269,7 @@ func resyncPhysicalRestores(
 		return nil
 	}
 
-	restoreMeta, err := getAllRestoreMetaFromStorage(ctx, stg, skipRestores)
+	restoreMeta, err := getAllRestoreMetaFromStorage(ctx, stg, includeRestores)
 	if err != nil {
 		return errors.Wrap(err, "get all restore meta from storage")
 	}
@@ -322,7 +322,7 @@ func getAllBackupMetaFromStorage(
 func getAllRestoreMetaFromStorage(
 	ctx context.Context,
 	stg storage.Storage,
-	skipRestores bool,
+	includeRestores bool,
 ) ([]*restore.RestoreMeta, error) {
 	l := log.LogEventFromContext(ctx)
 
@@ -331,19 +331,16 @@ func getAllRestoreMetaFromStorage(
 		return nil, errors.Wrap(err, "get physical restores list from the storage")
 	}
 
-	var targets []storage.FileInfo
-
-	if skipRestores && len(restoreMeta) > 0 {
+	targets := restoreMeta
+	if !includeRestores && len(restoreMeta) > 0 {
 		l.Debug("only processing last restore")
 		latest := restoreMeta[0]
-		for _, f := range restoreMeta[1:] {
+		for _, f := range restoreMeta {
 			if f.Name > latest.Name {
 				latest = f
 			}
 		}
 		targets = []storage.FileInfo{latest}
-	} else {
-		targets = restoreMeta
 	}
 
 	rv := make([]*restore.RestoreMeta, 0, len(targets))

sdk/impl.go

@@ -262,10 +262,10 @@ func (c *Client) GetRestoreByOpID(ctx context.Context, opid string) (*RestoreMet
 	return restore.GetRestoreMetaByOPID(ctx, c.conn, opid)
 }
 
-func (c *Client) SyncFromStorage(ctx context.Context, skipRestores bool) (CommandID, error) {
+func (c *Client) SyncFromStorage(ctx context.Context, includeRestores bool) (CommandID, error) {
 	var opts *ctrl.ResyncCmd
-	if skipRestores {
-		opts = &ctrl.ResyncCmd{SkipRestores: true}
+	if includeRestores {
+		opts = &ctrl.ResyncCmd{IncludeRestores: true}
 	}
 	opid, err := ctrl.SendResync(ctx, c.conn, opts)
 	return CommandID(opid.String()), err