feat(go, CLI): key cleanup using batches #SCD-687

Author: jablan

clients/cli/cmd/internal/pull_target.go

@@ -160,7 +160,7 @@ func (t *Target) GetTags() []string {
 }
 
 func TargetsFromConfig(config phrase.Config) (Targets, error) {
-	if config.Targets == nil || len(config.Targets) == 0 {
+	if config.Pull == nil || len(config.Pull) == 0 {
 		return nil, fmt.Errorf("no targets for download specified")
 	}
 
@@ -170,7 +170,7 @@ func TargetsFromConfig(config phrase.Config) (Targets, error) {
 
 	targets := viper.New()
 	targets.SetConfigType("yaml")
-	err := targets.ReadConfig(bytes.NewReader(config.Targets))
+	err := targets.ReadConfig(bytes.NewReader(config.Pull))
 
 	if err != nil {
 		return nil, err

clients/cli/cmd/internal/push.go

@@ -43,19 +43,20 @@ func (cmd *PushCommand) Run() error {
 		Debug = true
 	}
 
-	if cmd.Cleanup && !cmd.Wait {
-		return fmt.Errorf("You can only use the --cleanup option together with --wait")
-	}
-
 	Config = &cmd.Config
 
 	client := newClient()
 
-	sources, err := SourcesFromConfig(cmd.Config)
+	sources, deleteUnmentionedKeys, err := SourcesFromConfig(cmd.Config)
 	if err != nil {
 		return err
 	}
 
+	// Use delete_unmentioned_keys from config if Cleanup wasn't explicitly set via command line
+	if deleteUnmentionedKeys && !cmd.Cleanup {
+		cmd.Cleanup = deleteUnmentionedKeys
+	}
+
 	if err := sources.Validate(); err != nil {
 		return err
 	}
@@ -154,23 +155,21 @@ func (cmd *PushCommand) Run() error {
 		if err != nil {
 			return err
 		}
-		if cmd.Wait && cmd.Cleanup {
-			// collect all upload ids for cleanup by project and branch
-			found := false
-			for _, result := range pushResults {
-				if result.ProjectID == pushResult.ProjectID && result.Branch == pushResult.Branch {
-					result.UploadIDs = append(result.UploadIDs, pushResult.UploadIDs...)
-					found = true
-					break
-				}
-			}
-			if !found {
-				pushResults = append(pushResults, pushResult)
+		// collect all upload ids for cleanup by project and branch
+		found := false
+		for _, result := range pushResults {
+			if result.ProjectID == pushResult.ProjectID && result.Branch == pushResult.Branch {
+				result.UploadIDs = append(result.UploadIDs, pushResult.UploadIDs...)
+				found = true
+				break
 			}
 		}
+		if !found {
+			pushResults = append(pushResults, pushResult)
+		}
 	}
 	for _, pushResult := range pushResults {
-		UploadCleanup(client, true, pushResult.UploadIDs, pushResult.Branch, pushResult.ProjectID)
+		CreateUploadBatch(client, true, pushResult.UploadIDs, pushResult.Branch, pushResult.ProjectID, cmd.Cleanup)
 	}
 
 	return nil

clients/cli/cmd/internal/push_source.go

@@ -13,26 +13,27 @@ import (
 	"github.com/spf13/viper"
 )
 
-func SourcesFromConfig(config phrase.Config) (Sources, error) {
-	if config.Sources == nil || len(config.Sources) == 0 {
-		return nil, fmt.Errorf("no sources for upload specified")
+func SourcesFromConfig(config phrase.Config) (Sources, bool, error) {
+	if config.Push == nil || len(config.Push) == 0 {
+		return nil, false, fmt.Errorf("no sources for upload specified")
 	}
 
 	tmp := struct {
-		Sources Sources
+		DeleteUnmentionedKeys bool    `json:"delete_unmentioned_keys,omitempty"`
+		Sources               Sources `json:"sources,omitempty"`
 	}{}
 
-	sources := viper.New()
-	sources.SetConfigType("yaml")
-	err := sources.ReadConfig(bytes.NewReader(config.Sources))
+	pushSection := viper.New()
+	pushSection.SetConfigType("yaml")
+	err := pushSection.ReadConfig(bytes.NewReader(config.Push))
 
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
-	err = sources.UnmarshalExact(&tmp, ViperStructTag())
+	err = pushSection.UnmarshalExact(&tmp, ViperStructTag())
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	srcs := tmp.Sources
@@ -64,10 +65,10 @@ func SourcesFromConfig(config phrase.Config) (Sources, error) {
 	}
 
 	if len(validSources) <= 0 {
-		return nil, fmt.Errorf("no sources could be identified! Refine the sources list in your config")
+		return nil, false, fmt.Errorf("no sources could be identified! Refine the sources list in your config")
 	}
 
-	return validSources, nil
+	return validSources, tmp.DeleteUnmentionedKeys, nil
 }
 
 type Sources []*Source

clients/cli/cmd/internal/shared.go

@@ -210,49 +210,50 @@ func StringToInterface() mapstructure.DecodeHookFunc {
 	}
 }
 
-func UploadCleanup(client *phrase.APIClient, confirm bool, ids []string, branch string, projectId string) error {
-	if !shared.BatchMode {
-		fmt.Println("Keys not mentioned in the following uploads will be deleted:")
-		fmt.Println(strings.Join(ids, "\n"))
-	}
-	if !confirm {
-		if shared.BatchMode {
-			return errors.New("Can't ask for confirmation in batch mode. Aborting")
-		}
-		confirmation := ""
-		err := prompt.WithDefault("Are you sure you want to continue? (y/n)", &confirmation, "n")
-		if err != nil {
-			return err
+func CreateUploadBatch(client *phrase.APIClient, confirm bool, ids []string, branch string, projectId string, cleanup bool) error {
+	if cleanup {
+		if !shared.BatchMode {
+			fmt.Println("Keys not mentioned in the following uploads will be deleted:")
+			fmt.Println(strings.Join(ids, "\n"))
 		}
+		if !confirm {
+			if shared.BatchMode {
+				return errors.New("Can't ask for confirmation in batch mode. Aborting")
+			}
+			confirmation := ""
+			err := prompt.WithDefault("Are you sure you want to continue? (y/n)", &confirmation, "n")
+			if err != nil {
+				return err
+			}
 
-		if strings.ToLower(confirmation) != "y" {
-			fmt.Println("Clean up aborted")
-			return nil
+			if strings.ToLower(confirmation) != "y" {
+				fmt.Println("Clean up aborted")
+				return nil
+			}
 		}
 	}
 
-	q := "unmentioned_in_upload:" + strings.Join(ids, ",")
-	optionalBranch := optional.String{}
-	if branch != "" {
-		optionalBranch = optional.NewString(branch)
-	}
-	keysDeletelocalVarOptionals := phrase.KeysDeleteCollectionOpts{
-		Q:      optional.NewString(q),
-		Branch: optionalBranch,
+	uploadBatchesCreateParameters := phrase.UploadBatchesCreateParameters{
+		Branch:                branch,
+		DeleteUnmentionedKeys: &cleanup,
+		UploadIds:             ids,
 	}
-	affected, _, err := client.KeysApi.KeysDeleteCollection(Auth, projectId, &keysDeletelocalVarOptionals)
-
+	uploadBatch, _, err := client.UploadBatchesApi.UploadBatchesCreate(Auth, projectId, uploadBatchesCreateParameters, nil)
 	if err != nil {
 		return err
 	}
+	if !cleanup {
+		return nil
+	}
+
 	if shared.BatchMode {
-		jsonBuf, jsonErr := json.MarshalIndent(affected, "", " ")
+		jsonBuf, jsonErr := json.MarshalIndent(uploadBatch, "", " ")
 		if jsonErr != nil {
 			print.Error(jsonErr)
 		}
 		fmt.Printf("%s\n", string(jsonBuf))
 	} else {
-		print.Success("%d key(s) successfully deleted.\n", affected.RecordsAffected)
+		print.Success("Keys cleanup scheduled for an upload batch with %d uploads", len(ids))
 	}
 	return nil
 }

clients/cli/cmd/internal/upload_cleanup.go

@@ -16,5 +16,5 @@ func (cmd *UploadCleanupCommand) Run() error {
 	Config = &cmd.Config
 	client := newClient()
 
-	return UploadCleanup(client, cmd.Confirm, cmd.IDs, cmd.Branch, cmd.ProjectID)
+	return CreateUploadBatch(client, cmd.Confirm, cmd.IDs, cmd.Branch, cmd.ProjectID, true)
 }

clients/go/config.go

@@ -3,7 +3,6 @@ package phrase
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -32,8 +31,8 @@ type Config struct {
 
 	Defaults map[string]map[string]interface{}
 
-	Targets []byte
-	Sources []byte
+	Pull []byte
+	Push []byte
 
 	UserAgent string
 }
@@ -83,7 +82,7 @@ func configContent(cfgFilePath string) ([]byte, error) {
 	case path == "":
 		return nil, nil
 	default:
-		return ioutil.ReadFile(path)
+		return os.ReadFile(path)
 	}
 }
 
@@ -141,8 +140,8 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(i interface{}) error) error {
 		"per_page":     &cfg.PerPage,
 		"project_id":   &cfg.DefaultProjectID,
 		"file_format":  &cfg.DefaultFileFormat,
-		"push":         &cfg.Sources,
-		"pull":         &cfg.Targets,
+		"push":         &cfg.Push,
+		"pull":         &cfg.Pull,
 		"defaults":     &m,
 	})
 	if err != nil {