Handle "environments" folders for new GitOps directory structures (#73)

Adjust for /environments/env-name/ at top of GitOps repository structure.

Author: a-roberts

README.md

@@ -23,10 +23,35 @@ $ ./services promote --from https://github.com/organisation/first-environment.gi
 
 If the `commit-name` and `commit-email` are not provided, it will attempt to find them in `~/.gitconfig`, otherwise it will fail.
 
+
 This will _copy_ all files under `/services/service-a/base/config/*` in `first-environment` to `second-environment`, commit and push, and open a PR for the change.
 
+
+## Using environments 
+
+
+If an `environments` folder exists in the GitOps repository you are promoting into, and that only has one folder, the files will be copied into the destination repository's `/environments/<the only folder>` directory.
+
+Future support is planned for an `--env` like flag which will allow us to promote from/to different repositories with multiple environments.
+
 ## Testing
 
+Linting should be done first (this is done on Travis, and what's good locally should be good there too)
+
+Grab the linter if you haven't already: 
+
+```shell
+GO111MODULE=on go get github.com/golangci/golangci-lint/cmd/[email protected]
+```
+
+Then you can do:
+
+```shell
+golangci-lint run
+```
+
+Run the unit tests:
+
 ```shell
 $ go test ./...
 ```

go.mod

@@ -4,16 +4,15 @@ go 1.13
 
 require (
 	github.com/golang/protobuf v1.3.2 // indirect
+	github.com/golangci/golangci-lint v1.26.0 // indirect
 	github.com/google/go-cmp v0.3.0
 	github.com/google/uuid v1.1.1
 	github.com/jenkins-x/go-scm v1.5.77
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 // indirect
 	github.com/spf13/cobra v1.0.0
-	github.com/spf13/pflag v1.0.3
+	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.6.3
 	github.com/tcnksm/go-gitconfig v0.1.2
-	golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e // indirect
 )

go.sum

@@ -107,6 +107,7 @@ func (s *ServiceManager) Promote(serviceName, fromURL, toURL, newBranchName, mes
 	var localSource git.Source
 	var errorSource error
 	isLocal := fromLocalRepo(fromURL)
+
 	if isLocal {
 		localSource = s.localFactory(fromURL, s.debug)
 		if newBranchName == "" {
@@ -131,16 +132,38 @@ func (s *ServiceManager) Promote(serviceName, fromURL, toURL, newBranchName, mes
 		// This would be a checkout error as the clone error gives us the above gitError instead
 		return fmt.Errorf("failed to checkout destination repository, error: %w", err)
 	}
+
+	if destination == nil {
+		// Should never happen, but if it does...
+		return fmt.Errorf("destination repository was not initialised despite being no errors")
+	}
+
 	reposToDelete = append(reposToDelete, destination)
 
 	var copied []string
+
 	if isLocal {
-		copied, err = local.CopyConfig(serviceName, localSource, destination)
+		overrideTargetFolder, err := destination.GetUniqueEnvironmentFolder()
+		if err != nil {
+			return fmt.Errorf("could not determine unique environment name for destination repository - check that only one directory exists under it and you can write to your cache folder")
+		}
+		copied, err = local.CopyConfig(serviceName, localSource, destination, overrideTargetFolder)
 		if err != nil {
 			return fmt.Errorf("failed to set up local repository: %w", err)
 		}
 	} else {
-		copied, err = git.CopyService(serviceName, source, destination)
+		sourceEnvironment, err := source.GetUniqueEnvironmentFolder()
+		if err != nil {
+			return fmt.Errorf("could not determine unique environment name for source repository - check that only one directory exists under it and you can write to your cache folder")
+		}
+
+		destinationEnvironment, err := destination.GetUniqueEnvironmentFolder()
+
+		if err != nil {
+			return fmt.Errorf("could not determine unique environment name for destination repository - check that only one directory exists under it and you can write to your cache folder")
+		}
+
+		copied, err = git.CopyService(serviceName, source, destination, sourceEnvironment, destinationEnvironment)
 		if err != nil {
 			return fmt.Errorf("failed to copy service: %w", err)
 		}
@@ -149,14 +172,13 @@ func (s *ServiceManager) Promote(serviceName, fromURL, toURL, newBranchName, mes
 	commitMsg := message
 	if commitMsg == "" {
 		if isLocal {
-			commitMsg = fmt.Sprintf("Promotion of service `%s` from local filesystem directory `%s`.", serviceName, fromURL)
+			commitMsg = fmt.Sprintf("Promotion of service %s from local filesystem directory %s.", serviceName, fromURL)
 		} else {
 			commitMsg = generateDefaultCommitMsg(source, serviceName, fromURL, fromBranch)
 		}
 	}
-
 	if err := destination.StageFiles(copied...); err != nil {
-		return fmt.Errorf("failed to stage files: %w", err)
+		return fmt.Errorf("failed to stage files %s: %w", copied, err)
 	}
 	if err := destination.Commit(commitMsg, s.author); err != nil {
 		return fmt.Errorf("failed to commit: %w", err)
@@ -198,7 +220,7 @@ func (s *ServiceManager) checkoutDestinationRepo(repoURL, branch string) (git.Re
 	}
 	err = repo.CheckoutAndCreate(branch)
 	if err != nil {
-		return nil, fmt.Errorf("failed to checkout branch %s: %w", branch, err)
+		return nil, fmt.Errorf("failed to checkout branch %s, error: %w", branch, err)
 	}
 	return repo, nil
 }
@@ -280,6 +302,6 @@ func generateBranchForLocalSource(source git.Source) string {
 // generateDefaultCommitMsg constructs a default commit message based on the source information.
 func generateDefaultCommitMsg(sourceRepo git.Repo, serviceName, from, fromBranch string) string {
 	commit := sourceRepo.GetCommitID()
-	msg := fmt.Sprintf("Promoting service `%s` at commit `%s` from branch `%s` in `%s`.", serviceName, commit, fromBranch, from)
+	msg := fmt.Sprintf("Promoting service %s at commit %s from branch %s in %s.", serviceName, commit, fromBranch, from)
 	return msg
 }

pkg/avancement/service_manager.go

@@ -37,15 +37,15 @@ func TestPromoteWithSuccessCustomMsg(t *testing.T) {
 func promoteWithSuccess(t *testing.T, keepCache bool, repoType string, tlsVerify bool, msg string) {
 	dstBranch := "test-branch"
 	author := &git.Author{Name: "Testing User", Email: "[email protected]", Token: "test-token"}
-	devRepo, stagingRepo := mock.New("/dev", "master"), mock.New("/staging", "master")
+	devRepo, stagingRepo := mock.New("environments/dev", "master"), mock.New("environments/staging", "master")
 	repos := map[string]*mock.Repository{
 		mustAddCredentials(t, dev, author):     devRepo,
 		mustAddCredentials(t, staging, author): stagingRepo,
 	}
 	sm := New("tmp", author)
 	sm.repoType = repoType
 	sm.tlsVerify = tlsVerify
-	sm.clientFactory = func(s, ty, r  string, v bool) *scm.Client {
+	sm.clientFactory = func(s, ty, r string, v bool) *scm.Client {
 		client, _ := fakescm.NewDefault()
 		if r != repoType {
 			t.Fatalf("repoType doesn't match %s != %s\n", r, repoType)
@@ -61,8 +61,8 @@ func promoteWithSuccess(t *testing.T, keepCache bool, repoType string, tlsVerify
 		}
 		return git.Repo(repos[url]), nil
 	}
-	devRepo.AddFiles("/services/my-service/base/config/myfile.yaml")
-
+	devRepo.AddFiles("services/my-service/base/config/myfile.yaml")
+	stagingRepo.AddFiles("")
 	err := sm.Promote("my-service", dev, staging, dstBranch, msg, keepCache)
 	if err != nil {
 		t.Fatal(err)
@@ -71,11 +71,11 @@ func promoteWithSuccess(t *testing.T, keepCache bool, repoType string, tlsVerify
 	expectedCommitMsg := msg
 	if msg == "" {
 		commit := devRepo.GetCommitID()
-		expectedCommitMsg = fmt.Sprintf("Promoting service `my-service` at commit `%s` from branch `master` in `%s`.", commit, dev)
+		expectedCommitMsg = fmt.Sprintf("Promoting service my-service at commit %s from branch master in %s.", commit, dev)
 	}
 
 	stagingRepo.AssertBranchCreated(t, "master", dstBranch)
-	stagingRepo.AssertFileCopiedInBranch(t, dstBranch, "/dev/services/my-service/base/config/myfile.yaml", "/staging/services/my-service/base/config/myfile.yaml")
+	stagingRepo.AssertFileCopiedInBranch(t, dstBranch, "environments/dev/services/my-service/base/config/myfile.yaml", "environments/staging/services/my-service/base/config/myfile.yaml")
 	stagingRepo.AssertCommit(t, dstBranch, expectedCommitMsg, author)
 	stagingRepo.AssertPush(t, dstBranch)
 
@@ -103,11 +103,11 @@ func TestPromoteLocalWithSuccessCustomMsg(t *testing.T) {
 func promoteLocalWithSuccess(t *testing.T, keepCache bool, msg string) {
 	dstBranch := "test-branch"
 	author := &git.Author{Name: "Testing User", Email: "[email protected]", Token: "test-token"}
-	stagingRepo := mock.New("/staging", "master")
+	stagingRepo := mock.New("environments", "master")
 	devRepo := NewLocal("/dev")
 
 	sm := New("tmp", author)
-	sm.clientFactory = func(s, t, r  string, v bool) *scm.Client {
+	sm.clientFactory = func(s, t, r string, v bool) *scm.Client {
 		client, _ := fakescm.NewDefault()
 		return client
 	}
@@ -118,7 +118,8 @@ func promoteLocalWithSuccess(t *testing.T, keepCache bool, msg string) {
 		return git.Source(devRepo)
 	}
 	sm.debug = true
-	devRepo.AddFiles("/config/myfile.yaml")
+	devRepo.AddFiles("config/myfile.yaml")
+	stagingRepo.AddFiles("staging")
 
 	err := sm.Promote("my-service", ldev, staging, dstBranch, msg, keepCache)
 	if err != nil {
@@ -127,11 +128,11 @@ func promoteLocalWithSuccess(t *testing.T, keepCache bool, msg string) {
 
 	expectedCommitMsg := msg
 	if expectedCommitMsg == "" {
-		expectedCommitMsg = "Promotion of service `my-service` from local filesystem directory `/root/repo`."
+		expectedCommitMsg = "Promotion of service my-service from local filesystem directory /root/repo."
 	}
 
 	stagingRepo.AssertBranchCreated(t, "master", dstBranch)
-	stagingRepo.AssertFileCopiedInBranch(t, dstBranch, "/dev/config/myfile.yaml", "/staging/services/my-service/base/config/myfile.yaml")
+	stagingRepo.AssertFileCopiedInBranch(t, dstBranch, "/dev/config/myfile.yaml", "environments/staging/services/my-service/base/config/myfile.yaml")
 	stagingRepo.AssertCommit(t, dstBranch, expectedCommitMsg, author)
 	stagingRepo.AssertPush(t, dstBranch)
 
@@ -142,6 +143,70 @@ func promoteLocalWithSuccess(t *testing.T, keepCache bool, msg string) {
 	}
 }
 
+func TestPromoteLocalWithSuccessOneEnvAndIsUsed(t *testing.T) {
+	// Destination repo (GitOps repo) to have /environments/staging
+	// Promotion should copy files into that staging directory
+	dstBranch := "test-branch"
+	author := &git.Author{Name: "Testing User", Email: "[email protected]", Token: "test-token"}
+	stagingRepo := mock.New("environments", "master")
+	devRepo := NewLocal("/dev")
+
+	sm := New("tmp", author)
+	sm.clientFactory = func(s, t, r string, v bool) *scm.Client {
+		client, _ := fakescm.NewDefault()
+		return client
+	}
+	sm.repoFactory = func(url, _ string, _ bool, _ bool) (git.Repo, error) {
+		return git.Repo(stagingRepo), nil
+	}
+	sm.localFactory = func(path string, _ bool) git.Source {
+		return git.Source(devRepo)
+	}
+	sm.debug = true
+	devRepo.AddFiles("/config/myfile.yaml")
+	stagingRepo.AddFiles("/staging")
+
+	err := sm.Promote("my-service", ldev, staging, dstBranch, "", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedCommitMsg := "Promotion of service my-service from local filesystem directory /root/repo."
+
+	stagingRepo.AssertBranchCreated(t, "master", dstBranch)
+	stagingRepo.AssertFileCopiedInBranch(t, dstBranch, "/dev/config/myfile.yaml", "environments/staging/services/my-service/base/config/myfile.yaml")
+	stagingRepo.AssertCommit(t, dstBranch, expectedCommitMsg, author)
+	stagingRepo.AssertPush(t, dstBranch)
+}
+
+func TestPromoteErrorsIfMultipleEnvironments(t *testing.T) {
+	dstBranch := "test-branch"
+	author := &git.Author{Name: "Testing User", Email: "[email protected]", Token: "test-token"}
+	devRepo, stagingRepo := mock.New("/", "master"), mock.New("/environments", "master")
+
+	stagingRepo.AddFiles("/staging")
+	stagingRepo.AddFiles("/prod")
+
+	repos := map[string]*mock.Repository{
+		mustAddCredentials(t, dev, author):     devRepo,
+		mustAddCredentials(t, staging, author): stagingRepo,
+	}
+	sm := New("tmp", author)
+	sm.clientFactory = func(s, ty, r string, v bool) *scm.Client {
+		client, _ := fakescm.NewDefault()
+		return client
+	}
+	sm.repoFactory = func(url, _ string, v bool, _ bool) (git.Repo, error) {
+		return git.Repo(repos[url]), nil
+	}
+	devRepo.AddFiles("services/my-service/base/config/myfile.yaml")
+
+	msg := "foo message"
+	err := sm.Promote("my-service", dev, staging, dstBranch, msg, false)
+	if err == nil {
+		t.Fail()
+	}
+}
+
 func TestAddCredentials(t *testing.T) {
 	testUser := &git.Author{Name: "Test User", Email: "[email protected]", Token: "test-token"}
 	tests := []struct {
@@ -177,32 +242,33 @@ func mustAddCredentials(t *testing.T, repoURL string, a *git.Author) string {
 func TestPromoteWithCacheDeletionFailure(t *testing.T) {
 	dstBranch := "test-branch"
 	author := &git.Author{Name: "Testing User", Email: "[email protected]", Token: "test-token"}
-	devRepo, stagingRepo := mock.New("/dev", "master"), mock.New("/staging", "master")
+	devRepo, stagingRepo := mock.New("environments", "master"), mock.New("environments", "master")
 	stagingRepo.DeleteErr = errors.New("failed test delete")
 	repos := map[string]*mock.Repository{
 		mustAddCredentials(t, dev, author):     devRepo,
 		mustAddCredentials(t, staging, author): stagingRepo,
 	}
 	sm := New("tmp", author)
-	sm.clientFactory = func(s, t, r  string, v bool) *scm.Client {
+	sm.clientFactory = func(s, t, r string, v bool) *scm.Client {
 		client, _ := fakescm.NewDefault()
 		return client
 	}
 	sm.repoFactory = func(url, _ string, _ bool, _ bool) (git.Repo, error) {
 		return git.Repo(repos[url]), nil
 	}
-	devRepo.AddFiles("/services/my-service/base/config/myfile.yaml")
+	devRepo.AddFiles("dev/services/my-service/base/config/myfile.yaml")
+	stagingRepo.AddFiles("staging")
 
 	err := sm.Promote("my-service", dev, staging, dstBranch, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	commit := devRepo.GetCommitID()
-	expectedCommitMsg := fmt.Sprintf("Promoting service `my-service` at commit `%s` from branch `master` in `%s`.", commit, dev)
+	expectedCommitMsg := fmt.Sprintf("Promoting service my-service at commit %s from branch master in %s.", commit, dev)
 
 	stagingRepo.AssertBranchCreated(t, "master", dstBranch)
-	stagingRepo.AssertFileCopiedInBranch(t, dstBranch, "/dev/services/my-service/base/config/myfile.yaml", "/staging/services/my-service/base/config/myfile.yaml")
+	stagingRepo.AssertFileCopiedInBranch(t, dstBranch, "environments/dev/services/my-service/base/config/myfile.yaml", "environments/staging/services/my-service/base/config/myfile.yaml")
 	stagingRepo.AssertCommit(t, dstBranch, expectedCommitMsg, author)
 	stagingRepo.AssertPush(t, dstBranch)
 
@@ -298,7 +364,7 @@ func TestRepositoryCloneErrorOmitsToken(t *testing.T) {
 	dstBranch := "test-branch"
 	author := &git.Author{Name: "Testing User", Email: "[email protected]", Token: "test-token"}
 	client, _ := fakescm.NewDefault()
-	fakeClientFactory := func(s, t, r  string, v bool)  *scm.Client {
+	fakeClientFactory := func(s, t, r string, v bool) *scm.Client {
 		return client
 	}
 	sm := New("tmp", author)

pkg/avancement/service_manager_test.go

@@ -66,7 +66,6 @@ func Execute() {
 	logIfError(cobra.MarkFlagRequired(rootCmd.PersistentFlags(), githubTokenFlag))
 	logIfError(viper.BindPFlag(githubTokenFlag, rootCmd.PersistentFlags().Lookup(githubTokenFlag)))
 	rootCmd.AddCommand(makePromoteCmd())
-
 	if err := rootCmd.Execute(); err != nil {
 		log.Fatal(err)
 	}

pkg/cmd/root.go

@@ -12,16 +12,15 @@ import (
 // Only files under /services/[serviceName]/base/config/* are copied to the destination
 //
 // Returns the list of files that were copied, and possibly an error.
-func CopyService(serviceName string, source Source, dest Destination) ([]string, error) {
-
+func CopyService(serviceName string, source Source, dest Destination, sourceEnvironment, destinationEnvironment string) ([]string, error) {
 	// filePath defines the root folder for serviceName's config in the repository
-	filePath := pathForServiceConfig(serviceName)
-
+	// the lookup is done for the source repository
+	filePath := pathForServiceConfig(serviceName, sourceEnvironment)
 	copied := []string{}
 	err := source.Walk(filePath, func(prefix, name string) error {
 		sourcePath := path.Join(prefix, name)
-		destPath := pathForServiceConfig(name)
-		if pathValidForPromotion(serviceName, destPath) {
+		destPath := pathForServiceConfig(name, destinationEnvironment)
+		if pathValidForPromotion(serviceName, destPath, destinationEnvironment) {
 			err := dest.CopyFile(sourcePath, destPath)
 			if err == nil {
 				copied = append(copied, destPath)
@@ -30,19 +29,19 @@ func CopyService(serviceName string, source Source, dest Destination) ([]string,
 		}
 		return nil
 	})
+
 	return copied, err
 }
 
-// pathValidForPromotion()
-//  For a given serviceName, only files in services/serviceName/base/config/* are valid for promotion
-//
-func pathValidForPromotion(serviceName, filePath string) bool {
-	filterPath := filepath.Join(pathForServiceConfig(serviceName), "base", "config")
+//  For a given serviceName, only files in environments/envName/services/serviceName/base/config/* are valid for promotion
+func pathValidForPromotion(serviceName, filePath, environmentName string) bool {
+	filterPath := filepath.Join(pathForServiceConfig(serviceName, environmentName), "base", "config")
 	validPath := strings.HasPrefix(filePath, filterPath)
 	return validPath
 }
 
 // pathForServiceConfig defines where in a 'gitops' repository the config for a given service should live.
-func pathForServiceConfig(serviceName string) string {
-	return filepath.Join("services", serviceName)
+func pathForServiceConfig(serviceName, environmentName string) string {
+	pathForConfig := filepath.Join("environments", environmentName, "services", serviceName)
+	return pathForConfig
 }