Merge pull request #11 from GwonsooLee/snapshot_goroutine_and_add_fix_command

add fix command & apply goroutine for restoring snapshot

Author: GwonsooLee

.gitignore

@@ -4,3 +4,4 @@ out/
 *.iml
 .idea/
 .vscode/
+vendor

cmd/cmd/builder/flag.go

@@ -102,6 +102,26 @@ var FlagRegistry = []Flag{
 			"snapshot restore [repositoryID] [snapshotID] [indexName]",
 			"snapshot archive [repositoryID] [snapshotID]"},
 	},
+	{
+		Name:          "restore-tier",
+		Usage:         "Tier of restore job",
+		Value:         aws.String("Standard"),
+		DefValue:      "Standard",
+		FlagAddMethod: "StringVar",
+		DefinedOn: []string{
+			"snapshot restore [repositoryID] [snapshotID] [indexName]",
+		},
+	},
+	{
+		Name:          "max-concurrent-job",
+		Usage:         "Maximum number of concurrent jobs for restoring snapshot",
+		Value:         aws.Int64(constants.DefaultMaxConcurrentJob),
+		DefValue:      int64(constants.DefaultMaxConcurrentJob),
+		FlagAddMethod: "Int64Var",
+		DefinedOn: []string{
+			"snapshot restore [repositoryID] [snapshotID] [indexName]",
+		},
+	},
 }
 
 func (fl *Flag) flag() *pflag.Flag {

cmd/cmd/cmd.go

@@ -44,6 +44,7 @@ func NewRootCommand(out, stderr io.Writer) *cobra.Command {
 	rootCmd.AddCommand(NewCatCommand())
 	rootCmd.AddCommand(NewVersionCommand())
 	rootCmd.AddCommand(NewDiagCommand())
+	rootCmd.AddCommand(NewFixCommand())
 	rootCmd.AddCommand(NewIndexCommand())
 	rootCmd.AddCommand(NewClusterCommand())
 	rootCmd.AddCommand(NewUpdateCommand())

cmd/cmd/fix.go

@@ -0,0 +1,39 @@
+/*
+copyright 2020 the Escli authors
+
+licensed under the apache license, version 2.0 (the "license");
+you may not use this file except in compliance with the license.
+you may obtain a copy of the license at
+
+    http://www.apache.org/licenses/license-2.0
+
+unless required by applicable law or agreed to in writing, software
+distributed under the license is distributed on an "as is" basis,
+without warranties or conditions of any kind, either express or implied.
+see the license for the specific language governing permissions and
+limitations under the license.
+*/
+
+package cmd
+
+import (
+	"context"
+	"io"
+
+	"github.com/spf13/cobra"
+
+	"github.com/DevopsArtFactory/escli/cmd/cmd/builder"
+	"github.com/DevopsArtFactory/escli/internal/executor"
+)
+
+func NewFixCommand() *cobra.Command {
+	return builder.NewCmd("fix").
+		WithDescription("fix old configuration or settings of escli yaml file").
+		NoArgs(funcFixCommand)
+}
+
+func funcFixCommand(ctx context.Context, out io.Writer) error {
+	return executor.RunExecutorWithoutCheckingConfig(ctx, func(executor executor.Executor) error {
+		return executor.Runner.Fix(out)
+	})
+}

go.mod

@@ -1,12 +1,13 @@
 module github.com/DevopsArtFactory/escli
 
-go 1.14
+go 1.16
 
 require (
 	github.com/AlecAivazis/survey/v2 v2.0.5
 	github.com/GoogleContainerTools/skaffold v1.17.2 // indirect
 	github.com/aws/aws-sdk-go v1.36.5
 	github.com/blang/semver v3.5.1+incompatible
+	github.com/cheggaaa/pb/v3 v3.0.8
 	github.com/elastic/go-elasticsearch v0.0.0
 	github.com/enescakir/emoji v1.0.0
 	github.com/fatih/color v1.10.0
@@ -17,6 +18,6 @@ require (
 	github.com/spf13/cobra v1.1.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.7.1
-	gopkg.in/yaml.v2 v2.3.0
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
 )

go.sum

@@ -95,6 +95,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
 github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
@@ -162,6 +164,8 @@ github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tj
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
 github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
+github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA=
+github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -571,6 +575,8 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow=
+github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
@@ -723,6 +729,9 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
 github.com/rhysd/go-github-selfupdate v1.2.3 h1:iaa+J202f+Nc+A8zi75uccC8Wg3omaM7HDeimXA22Ag=
 github.com/rhysd/go-github-selfupdate v1.2.3/go.mod h1:mp/N8zj6jFfBQy/XMYoWsmfzxazpPAODuqarmPDe2Rg=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
@@ -1071,6 +1080,8 @@ golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211 h1:9UQO31fZ+0aKQOFldThf7BKPMJTiBfWycGh/u3UoO88=
 golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/builder/builder.go

@@ -26,12 +26,14 @@ import (
 )
 
 type Flags struct {
-	Force        bool   `json:"force"`
-	TroubledOnly bool   `json:"troubled-only"`
-	RepoOnly     bool   `json:"repo-only"`
-	WithRepo     string `json:"with-repo"`
-	Region       string `json:"region"`
-	SortBy       string `json:"sort-by"`
+	Force            bool   `json:"force"`
+	TroubledOnly     bool   `json:"troubled-only"`
+	RepoOnly         bool   `json:"repo-only"`
+	WithRepo         string `json:"with-repo"`
+	Region           string `json:"region"`
+	SortBy           string `json:"sort-by"`
+	RestoreTier      string `json:"restore-tier"`
+	MaxConcurrentJob int64  `json:"max-concurrent-job"`
 }
 
 func ParseFlags() (*Flags, error) {

internal/client/s3.go

@@ -64,14 +64,14 @@ func (c Client) HeadObject(bucket *string, key *string) (*s3.HeadObjectOutput, e
 	return resp, nil
 }
 
-func (c Client) RestoreObject(bucket *string, key *string) error {
+func (c Client) RestoreObject(bucket *string, key *string, restoreTier string) error {
 	_, err := c.S3Client.RestoreObject(&s3.RestoreObjectInput{
 		Bucket: bucket,
 		Key:    key,
 		RestoreRequest: &s3.RestoreRequest{
 			Days: aws.Int64(10),
 			GlacierJobParameters: &s3.GlacierJobParameters{
-				Tier: aws.String("Standard"),
+				Tier: aws.String(restoreTier),
 			},
 		},
 	})

internal/config/config.go

@@ -17,24 +17,31 @@ limitations under the license.
 package config
 
 import (
+	"fmt"
 	"io/ioutil"
 
+	"github.com/fatih/color"
 	"github.com/spf13/viper"
 	"gopkg.in/yaml.v3"
 
 	"github.com/DevopsArtFactory/escli/internal/schema"
+	"github.com/DevopsArtFactory/escli/internal/util"
 )
 
 func GetConfig() (*schema.Config, error) {
 	var configs []schema.Config
 
-	yamlFile, err := ioutil.ReadFile(viper.GetString("cfgFile"))
+	filePath := viper.GetString("cfgFile")
+	yamlFile, err := ioutil.ReadFile(filePath)
 	if err != nil {
 		return nil, err
 	}
 
 	err = yaml.Unmarshal(yamlFile, &configs)
 	if err != nil {
+		if old, err := detectOldConfig(); err == nil && old {
+			return nil, fmt.Errorf("you are using old version configuration. please run `escli fix` to migrate configuration")
+		}
 		return nil, err
 	}
 
@@ -49,6 +56,78 @@ func GetConfig() (*schema.Config, error) {
 	return &configs[0], nil
 }
 
+func detectConfig() (bool, error) {
+	var config []schema.Config
+	filePath := viper.GetString("cfgFile")
+	yamlFile, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return false, err
+	}
+	err = yaml.Unmarshal(yamlFile, &config)
+	if err != nil {
+		return false, err
+	}
+
+	return true, err
+}
+
+func detectOldConfig() (bool, error) {
+	var config schema.OldConfig
+	filePath := viper.GetString("cfgFile")
+	yamlFile, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return false, err
+	}
+	err = yaml.Unmarshal(yamlFile, &config)
+	if err != nil {
+		return false, err
+	}
+
+	return true, err
+}
+
+func GetOldConfig() (*schema.OldConfig, error) {
+	var config schema.OldConfig
+
+	filePath := viper.GetString("cfgFile")
+	yamlFile, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(yamlFile, &config)
+	if err != nil {
+		if new, err := detectConfig(); err == nil && new {
+			color.Green("your configuration is already updated")
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	return &config, nil
+}
+
+func ConvertToNewConfig(oc *schema.OldConfig) error {
+	cfgFile := viper.GetString("cfgFile")
+
+	profile, err := util.AskProfile()
+	if err != nil {
+		return err
+	}
+
+	c := SetInitConfig(profile, oc.ElasticSearchURL, oc.AWSRegion)
+	y, err := yaml.Marshal(c)
+	if err != nil {
+		return err
+	}
+
+	if err := util.CreateFile(cfgFile, string(y)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func GetDefaultConfig() (*schema.Config, error) {
 	profile := &schema.Config{}
 

internal/constants/constants.go

@@ -30,11 +30,15 @@ const (
 
 	GetClusterSetting = 0
 	PutClusterSetting = 3
+
+	HardLimitMaxConcurrentJob = 100
+	DefaultMaxConcurrentJob   = 50
 )
 
 var (
 	ConfigDirectoryPath = HomeDir() + "/.escli"
 	BaseFilePath        = ConfigDirectoryPath + "/config.yaml"
+	ValidRestoreTier    = []string{"Standard", "Bulk", "Expedited"}
 )
 
 // Get Home Directory