Merge pull request #6 from etsy/ay/enable-create-gitops-pr-rule

Enable create_gitops_prs rule again

Author: jumbosushi

README.md

@@ -372,6 +372,8 @@ For the full list of `create_gitops_prs` command line options, run:
 bazel run @rules_gitops//gitops/prer:create_gitops_prs
 ```
 
+Alternatively, you can use the `create_gitops_prs` rule that references `gitops` targets. You can run the target to template yaml & image push in parallel.
+
 <a name="gitops-and-deployment-supported-git-servers"></a>
 ### Supported Git Servers
 

gitops/create_gitops_prs.tpl.sh

@@ -2,6 +2,7 @@
 set -x
 set -e
 
-GIT_COMMIT=$(git rev-parse HEAD)
+CURR_GIT_COMMIT=$(git rev-parse HEAD)
+GIT_COMMIT=${BUILDKITE_COMMIT:-$CURR_GIT_COMMIT}
 
-%{prer} --git_commit=$GIT_COMMIT %{params} "${@}"
+%{prer} --git_commit $GIT_COMMIT %{params}

gitops/gitops.bzl

@@ -40,6 +40,15 @@ def __create_gitops_prs_impl(ctx):
         params += "--github_repo_owner {} ".format(ctx.attr.github_repo_owner)
     if ctx.attr.github_repo:
         params += "--github_repo {} ".format(ctx.attr.github_repo)
+    # Etsy Custom params
+    if ctx.attr.github_app_id:
+        params += "--github_app_id {} ".format(ctx.attr.github_app_id)
+    if ctx.attr.github_installation_id:
+        params += "--github_installation_id {} ".format(ctx.attr.github_installation_id)
+    if ctx.attr.github_app_repo_owner:
+        params += "--github_app_repo_owner {} ".format(ctx.attr.github_app_repo_owner)
+    if ctx.attr.private_key:
+        params += "--private_key {} ".format(ctx.attr.private_key)
 
     ctx.actions.expand_template(
         template = ctx.file._tpl,
@@ -127,6 +136,19 @@ create_gitops_prs = rule(
             cfg = "exec",
             executable = True,
         ),
+        # Etsy Custom params
+        "github_app_id": attr.string(
+            doc = "GitHub App Id"
+        ),
+        "github_installation_id": attr.string(
+            doc = "GitHub App Installation Id"
+        ),
+        "github_app_repo_owner": attr.string(
+            doc = "the owner user/organization to use for github api requests"
+        ),
+        "private_key": attr.string(
+            doc = "Private Key"
+        ),
     },
     executable = True,
 )

gitops/prer/create_gitops_prs.go

@@ -58,6 +58,10 @@ type Config struct {
 	PRBody                 string
 	DeploymentBranchSuffix string
 
+	// create_gitops_prs rule
+	ResolvedBinaries SliceFlags
+	ResolvedPushes   SliceFlags
+
 	// Dependencies
 	DependencyKinds []string
 	DependencyNames []string
@@ -92,6 +96,10 @@ func initConfig() *Config {
 	flag.StringVar(&cfg.PRBody, "gitops_pr_body", "", "PR body message")
 	flag.StringVar(&cfg.DeploymentBranchSuffix, "deployment_branch_suffix", "", "Suffix for deployment branch names")
 
+	// create_gitops_prs rule sets these when used with `bazel run`
+	flag.Var(&cfg.ResolvedBinaries, "resolved_binary", "list of resolved gitops binaries to run. Can be specified multiple times. format is releasetrain:cmd/binary/to/run/command. Default is empty")
+	flag.Var(&cfg.ResolvedPushes, "resolved_push", "list of resolved push binaries to run. Can be specified multiple times. format is cmd/binary/to/run/command. Default is empty")
+
 	// Dependencies
 	var kinds, names, attrs SliceFlags
 	flag.Var(&kinds, "gitops_dependencies_kind", "Dependency kinds for GitOps phase")
@@ -145,6 +153,27 @@ func executeBazelQuery(query string) *analysis.CqueryResult {
 	return result
 }
 
+func processResolvedImages(cfg *Config) {
+	resolvedPushChan := make(chan string)
+	var wg sync.WaitGroup
+	wg.Add(cfg.PushParallelism)
+
+	for i := 0; i < cfg.PushParallelism; i++ {
+		go func() {
+			defer wg.Done()
+			for cmd := range resolvedPushChan {
+				exec.Mustex("", cmd)
+			}
+		}()
+	}
+
+	for _, r := range cfg.ResolvedPushes {
+		resolvedPushChan <- r
+	}
+	close(resolvedPushChan)
+	wg.Wait()
+}
+
 func processImages(targets []string, cfg *Config) {
 	deps := fmt.Sprintf("set('%s')", strings.Join(targets, "' '"))
 	queries := []string{}
@@ -231,17 +260,30 @@ func main() {
 		}
 	}
 
-	// Find release trains
-	query := fmt.Sprintf("attr(deployment_branch, \".+\", attr(release_branch_prefix, \"%s\", kind(gitops, %s)))",
-		cfg.ReleaseBranch, cfg.Targets)
+	trains := make(map[string][]string)
+	if len(cfg.ResolvedBinaries) > 0 {
+		// This condition is used when calling the script from create_gitops_pr rules
+		// When you call `bazel run <create_gitops_pr target>`, you can't call another bazel query within a bazel run command
+		// So we have to rely on resolved binaries that were passed in
+		for _, rb := range cfg.ResolvedBinaries {
+			releaseTrain, bin, found := strings.Cut(rb, ":")
+			if !found {
+				log.Fatalf("resolved_binaries: invalid resolved_binary format: %s", rb)
+			}
+			trains[releaseTrain] = append(trains[releaseTrain], bin)
+		}
+	} else {
+		// Find release trains
+		query := fmt.Sprintf("attr(deployment_branch, \".+\", attr(release_branch_prefix, \"%s\", kind(gitops, %s)))",
+			cfg.ReleaseBranch, cfg.Targets)
 
-	result := executeBazelQuery(query)
+		result := executeBazelQuery(query)
 
-	trains := make(map[string][]string)
-	for _, t := range result.Results {
-		for _, attr := range t.Target.GetRule().GetAttribute() {
-			if attr.GetName() == "deployment_branch" {
-				trains[attr.GetStringValue()] = append(trains[attr.GetStringValue()], t.Target.Rule.GetName())
+		for _, t := range result.Results {
+			for _, attr := range t.Target.GetRule().GetAttribute() {
+				if attr.GetName() == "deployment_branch" {
+					trains[attr.GetStringValue()] = append(trains[attr.GetStringValue()], t.Target.Rule.GetName())
+				}
 			}
 		}
 	}
@@ -317,7 +359,11 @@ func main() {
 		return
 	}
 
-	processImages(updatedTargets, cfg)
+	if len(cfg.ResolvedPushes) > 0 {
+		processResolvedImages(cfg)
+	} else {
+		processImages(updatedTargets, cfg)
+	}
 
 	if !cfg.DryRun {
 		slug := os.Getenv("BUILDKITE_PIPELINE_SLUG")