Clean flag added with destinations support (#14)

* Implemention of Clean flag
---------

Co-authored-by: Rafal Przybyla <[email protected]>
Co-authored-by: Nathaniel Ritholtz <[email protected]>

Author: 3 people

go.mod

@@ -7,12 +7,13 @@ require (
 	github.com/nritholtz/stdemuxerhook v0.0.0-20181016194454-2c86ca05d211
 	github.com/rendon/testcli v0.0.0-20161027181003-6283090d169f
 	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.2.2
+	github.com/stretchr/testify v1.8.1
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,3 +1,4 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
@@ -10,8 +11,14 @@ github.com/rendon/testcli v0.0.0-20161027181003-6283090d169f h1:onGP+qmYmjKs7pkm
 github.com/rendon/testcli v0.0.0-20161027181003-6283090d169f/go.mod h1:cq57a4l475CeMvE7RRpSui1MEqCmhirIt1E7kl8BC2Q=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68 h1:z8Hj/bl9cOV2grsOpEaQFUaly0JWN3i97mo3jXKJNp0=
@@ -20,3 +27,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -39,6 +39,8 @@ var opts struct {
 	ModulePath string `short:"p" long:"module_path" default:"./vendor/modules" description:"File path to install generated terraform modules, if not overridden by 'destinations:' field"`
 
 	TerrafilePath string `short:"f" long:"terrafile_file" default:"./Terrafile" description:"File path to the Terrafile file"`
+
+	Clean bool `short:"c" long:"clean" description:"Remove everything from destinations and module path upon fetching module(s)\n !!! WARNING !!! Removes all files and folders in the destinations including non-modules."`
 }
 
 // To be set by goreleaser on build
@@ -93,6 +95,10 @@ func main() {
 		log.Fatalf("failed to parse yaml file due to error: %s", err)
 	}
 
+	if opts.Clean {
+		cleanDestinations(config)
+	}
+
 	// Clone modules
 	var wg sync.WaitGroup
 	_ = os.RemoveAll(opts.ModulePath)
@@ -157,3 +163,32 @@ func main() {
 
 	wg.Wait()
 }
+
+func cleanDestinations(config map[string]module) {
+
+	// Map filters duplicate destinations with key being each destination's file path
+	uniqueDestinations := make(map[string]bool)
+
+	// Range over config and gather all unique destinations
+	for _, m := range config {
+		if len(m.Destinations) == 0 {
+			uniqueDestinations[opts.ModulePath] = true
+			continue
+		}
+
+		// range over Destinations and put them into map
+		for _, dst := range m.Destinations {
+			// Destination supposed to be conjunction of destination defined in file with module path
+			d := filepath.Join(dst, opts.ModulePath)
+			uniqueDestinations[d] = true
+		}
+	}
+
+	for dst := range uniqueDestinations {
+
+		log.Infof("[*] Removing artifacts from %s", dst)
+		if err := os.RemoveAll(dst); err != nil {
+			log.Errorf("Failed to remove artifacts from %s due to error: %s", dst, err)
+		}
+	}
+}

main_test.go

@@ -63,6 +63,18 @@ func TestTerraformWithTerrafilePath(t *testing.T) {
 	} {
 		assert.Contains(t, testcli.Stdout(), output)
 	}
+
+	// Assert output does not contain the following
+	for _, output := range []string{
+		"[*] Removing artifacts from ./vendor/modules",
+		"[*] Removing artifacts from testdata/networking/vendor/modules",
+		"[*] Removing artifacts from testdata/some-other-stack/vendor/modules",
+		"[*] Removing artifacts from testdata/iam/vendor/modules",
+		"[*] Removing artifacts from testdata/onboarding/vendor/modules",
+	} {
+		assert.NotContains(t, testcli.Stdout(), output)
+	}
+
 	// Assert folder exist with default destination
 	for _, moduleName := range []string{
 		"tf-aws-vpc",
@@ -174,6 +186,81 @@ func TestTerraformWithTerrafilePath(t *testing.T) {
 	}
 }
 
+func TestTerraformWithTerrafileClean(t *testing.T) {
+	folder, back := setup(t)
+	defer back()
+
+	// Run original Terrafile with path
+	testcli.Run(terrafileBinaryPath, "-f", fmt.Sprint(folder, "/Terrafile"))
+
+	defer println(testcli.Stdout())
+	defer println(testcli.Stderr())
+
+	defer func() {
+		assert.NoError(t, os.RemoveAll("testdata/"))
+	}()
+
+	// Create Terrafile2.yaml
+	createTerrafile2(t, folder)
+
+	// Run terrafile with path to Terrafile2 and clean flag
+	testcli.Run(terrafileBinaryPath, "-f", fmt.Sprint(folder, "/Terrafile2"), "-c")
+
+	// Print out output of second run
+	defer println(testcli.Stdout())
+	defer println(testcli.Stderr())
+
+	if !testcli.Success() {
+		t.Fatalf("Expected to succeed, but failed: %q with message: %q", testcli.Error(), testcli.Stderr())
+	}
+
+	// Assert output
+	for _, output := range []string{
+		"[*] Removing artifacts from ./vendor/modules",
+		"[*] Removing artifacts from testdata/networking/vendor/modules",
+		"[*] Removing artifacts from testdata/some-other-stack/vendor/modules",
+		"[*] Removing artifacts from testdata/iam/vendor/modules",
+		"[*] Removing artifacts from testdata/onboarding/vendor/modules",
+	} {
+		assert.Contains(t, testcli.Stdout(), output)
+	}
+
+	// Assert files exist with default destination
+	for _, moduleName := range []string{
+		"tf-aws-vpc/main.tf",
+	} {
+		assert.FileExists(t, path.Join(workingDirectory, "vendor/modules", moduleName))
+	}
+
+	// Assert files does not exist with default destination
+	for _, moduleName := range []string{
+		"tf-aws-vpc-experimental/main.tf",
+	} {
+		assert.NoFileExists(t, path.Join(workingDirectory, "vendor/modules", moduleName))
+	}
+
+	// Assert files exist with non-default destinations
+	for _, moduleName := range []string{
+		"testdata/networking/vendor/modules/tf-aws-vpn-gateway/main.tf",
+
+		// terraform-aws-modules/terraform-aws-iam doesn't have main.tf, as it represents set of modules
+		// However, some terraform-aws-modules/terraform-aws-iam/modules have, e.g.:
+		"testdata/iam/vendor/modules/tf-aws-iam/modules/iam-account/main.tf",
+		"testdata/onboarding/vendor/modules/tf-aws-s3-bucket/main.tf",
+		"testdata/some-other-stack/vendor/modules/tf-aws-vpn-gateway/main.tf",
+	} {
+		assert.FileExists(t, path.Join(workingDirectory, moduleName))
+	}
+
+	// Assert files do not exist with non-default destinations
+	for _, moduleName := range []string{
+		"testdata/networking/vendor/modules/tf-aws-s3-bucket/main.tf",
+		"testdata/some-other-stack/vendor/modules/tf-aws-s3-bucket/main.tf",
+	} {
+		assert.NoFileExists(t, path.Join(workingDirectory, moduleName))
+	}
+}
+
 func setup(t *testing.T) (current string, back func()) {
 	folder, err := os.MkdirTemp("", "")
 	assert.NoError(t, err)
@@ -214,3 +301,27 @@ tf-aws-s3-bucket:
 `
 	createFile(t, path.Join(folder, "Terrafile"), yaml)
 }
+
+func createTerrafile2(t *testing.T, folder string) {
+	var yaml = `tf-aws-vpc:
+  source:  "[email protected]:terraform-aws-modules/terraform-aws-vpc"
+  version: "v1.46.0"
+tf-aws-vpn-gateway:
+  source:  "[email protected]:terraform-aws-modules/terraform-aws-vpn-gateway"
+  version: "v3.2.0"
+  destinations:
+    - testdata/networking
+    - testdata/some-other-stack
+tf-aws-iam:
+  source:  "[email protected]:terraform-aws-modules/terraform-aws-iam"
+  version: "v5.11.1"
+  destinations:
+    - testdata/iam
+tf-aws-s3-bucket:
+  source:  "[email protected]:terraform-aws-modules/terraform-aws-s3-bucket"
+  version: "v3.6.1"
+  destinations:
+    - testdata/onboarding
+`
+	createFile(t, path.Join(folder, "Terrafile2"), yaml)
+}