release: merge ER commits to RC branches

Previously, as a part of post-publishing tasks, we created merge commits
to the main release branch. When we have an extraordinary release, there
is a chance of having the next release's branch already cut. If the
extraordinary release PRs are not backported to the next release's
branch, we may have a regression, when the next version doesn't have
the fix merged to the extraordinary release branch.

This PR adds an additional check to verify that all changes from the
extraordinary branch are backported to the next release branch if it's
already cut. If there are changes, automation creates a merge PR to have
all extraordinary changes in the next release branch.

Additionally, add a check if there were no changes on the baking/staging
branch after it was cut and skip the merge PR creation.

Added a global flag pointing to the artifacts directory for cases we
need to store some outputs.

Fixes: RE-817
Release note: None

Author: rail

build/teamcity/internal/cockroach/release/process/update_versions_impl.sh

@@ -37,4 +37,5 @@ $(bazel info --config=crosslinux bazel-bin)/pkg/cmd/release/release_/release \
   [email protected] \
   --smtp-host=smtp.gmail.com \
   --smtp-port=587 \
+  --artifacts-dir=/artifacts \
   --to=$to

pkg/cmd/release/git.go

@@ -7,6 +7,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log"
 	"os/exec"
@@ -288,7 +289,7 @@ func listRemoteBranches(pattern string) ([]string, error) {
 	if err != nil {
 		return []string{}, fmt.Errorf("git ls-remote: %w", err)
 	}
-	log.Printf("git ls-remote returned: %s", out)
+	log.Printf("git ls-remote for %s returned: %s", pattern, out)
 	var remoteBranches []string
 	// Example output:
 	// $ git ls-remote origin "refs/heads/release-23.1*"
@@ -307,25 +308,102 @@ func listRemoteBranches(pattern string) ([]string, error) {
 	return remoteBranches, nil
 }
 
-// fileExistsInGit checks if a file exists in a local repository, assuming the remote name is `origin`.
-func fileExistsInGit(branch string, f string) (bool, error) {
-	cmd := exec.Command("git", "ls-tree", remoteOrigin+"/"+branch, f)
+// fileContent uses `git cat-file -p ref:file` to get to the file contents without `git checkout`.
+func fileContent(ref string, f string) (string, error) {
+	cmd := exec.Command("git", "cat-file", "-p", ref+":"+f)
 	out, err := cmd.Output()
 	if err != nil {
-		return false, fmt.Errorf("git ls-tree: %s %s %w, `%s`", branch, f, err, out)
+		return "", fmt.Errorf("git cat-file %s:%s: %w, `%s`", ref, f, err, out)
 	}
-	if len(out) == 0 {
-		return false, nil
+	return string(out), nil
+}
+
+// isAncestor checks if ref1 is an ancestor of ref2.
+// Returns true if ref1 is an ancestor of ref2, false if not, and error if the command fails.
+func isAncestor(ref1, ref2 string) (bool, error) {
+	cmd := exec.Command("git", "merge-base", "--is-ancestor", remoteOrigin+"/"+ref1, remoteOrigin+"/"+ref2)
+	err := cmd.Run()
+	if err != nil {
+		// Treat exit code 1 as false, as it means that ref1 is not an ancestor of ref2.
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 {
+			return false, nil
+		}
+		return false, fmt.Errorf("checking ancestry relationship between %s and %s: %w", ref1, ref2, err)
 	}
 	return true, nil
 }
 
-// fileContent uses `git cat-file -p ref:file` to get to the file contents without `git checkout`.
-func fileContent(ref string, f string) (string, error) {
-	cmd := exec.Command("git", "cat-file", "-p", ref+":"+f)
-	out, err := cmd.Output()
+// mergeCreatesContentChanges checks if a merge commit introduces changes to the branch.
+// Returns true if the merge commit introduces changes, false if not, and error if the command fails.
+func mergeCreatesContentChanges(branch, intoBranch string, ignoredPatterns []string) (bool, error) {
+	// Make sure the working directory is clean
+	statusCmd := exec.Command("git", "status", "--porcelain")
+	if out, err := statusCmd.Output(); err != nil {
+		return false, fmt.Errorf("checking git status: %w", err)
+	} else if len(out) > 0 {
+		return false, fmt.Errorf("working directory is not clean")
+	}
+
+	// Get the current branch name before we start
+	currentBranchCmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD")
+	currentBranch, err := currentBranchCmd.Output()
 	if err != nil {
-		return "", fmt.Errorf("git cat-file %s:%s: %w, `%s`", ref, f, err, out)
+		return false, fmt.Errorf("getting current branch: %w", err)
 	}
-	return string(out), nil
+	originalBranch := strings.TrimSpace(string(currentBranch))
+
+	// Checkout the branch to merge into. Use a temporary branch to avoid
+	// conflicts with the current branch name.
+	tmpIntoBranch := intoBranch + "-tmp"
+	checkoutCmd := exec.Command("git", "checkout", "-b", tmpIntoBranch, remoteOrigin+"/"+intoBranch)
+	if err := checkoutCmd.Run(); err != nil {
+		return false, fmt.Errorf("running checkout: %w", err)
+	}
+
+	// Run the merge command without committing and without fast-forward. If
+	// fast-forward is allowed and the current branch can fast forward, there
+	// will be no merge commit, so the --no-commit option won't work.
+	// We need to use the ours strategy to avoid conflicts. In the next step we
+	// will checkout the ignored files from the current branch (like version.txt).
+	mergeCmd := exec.Command("git", "merge", "--no-commit", "--no-ff", "--strategy=recursive", "-X", "ours", remoteOrigin+"/"+branch)
+	if err := mergeCmd.Run(); err != nil {
+		return false, fmt.Errorf("running merge: %w", err)
+	}
+	if len(ignoredPatterns) > 0 {
+		coCmd := exec.Command("git", "checkout", tmpIntoBranch, "--")
+		coCmd.Args = append(coCmd.Args, ignoredPatterns...)
+
+		if err := coCmd.Run(); err != nil {
+			return false, fmt.Errorf("running checkout: %w", err)
+		}
+	}
+
+	// Check if there are any content changes. The exit code will be analyzed to
+	// determine if there are changes after we clean up the current repo.
+	diffCmd := exec.Command("git", "diff", "--staged", "--quiet")
+	diffErr := diffCmd.Run()
+
+	// Always abort the merge attempt to clean up
+	if err := exec.Command("git", "merge", "--abort").Run(); err != nil {
+		return false, fmt.Errorf("aborting merge: %w", err)
+	}
+	if err := exec.Command("git", "checkout", originalBranch).Run(); err != nil {
+		return false, fmt.Errorf("running original branch checkout: %w", err)
+	}
+	if err := exec.Command("git", "branch", "-D", tmpIntoBranch).Run(); err != nil {
+		return false, fmt.Errorf("deleting tmp branch: %w", err)
+	}
+
+	// If diff returns no error (exit code 0), there are no changes
+	// If diff returns error with exit code 1, there are changes
+	// Any other error is unexpected
+	if diffErr == nil {
+		return false, nil // No changes
+	}
+	var exitErr *exec.ExitError
+	if errors.As(diffErr, &exitErr) && exitErr.ExitCode() == 1 {
+		return true, nil // Has changes
+	}
+	return false, fmt.Errorf("checking diff: %w", diffErr)
 }

pkg/cmd/release/main.go

@@ -8,6 +8,11 @@ package main
 import "github.com/spf13/cobra"
 
 var rootCmd = &cobra.Command{Use: "release"}
+var artifactsDir string
+
+func init() {
+	rootCmd.PersistentFlags().StringVar(&artifactsDir, "artifacts-dir", "", "artifacts directory")
+}
 
 const (
 	envSMTPUser     = "SMTP_USER"

pkg/cmd/release/update_versions.go

@@ -102,6 +102,10 @@ type prRepo struct {
 	workOnRepoError error
 }
 
+type metadata struct {
+	PRs []string `json:"prs"`
+}
+
 func (r prRepo) String() string {
 	return r.owner + "/" + r.repo + "@" + r.branch
 }
@@ -335,12 +339,32 @@ func updateVersions(_ *cobra.Command, _ []string) error {
 		workOnRepoErrors = append(workOnRepoErrors, err)
 		log.Printf("%s", err)
 	}
+	if artifactsDir != "" {
+		if err := saveMetadata(artifactsDir, metadata{PRs: prs}); err != nil {
+			err = fmt.Errorf("error saving metadata: %w", err)
+			workOnRepoErrors = append(workOnRepoErrors, err)
+			log.Printf("%s", err)
+		}
+	}
 	if len(workOnRepoErrors) > 0 {
 		return errors.Join(workOnRepoErrors...)
 	}
 	return nil
 }
 
+func saveMetadata(dir string, meta metadata) error {
+	dest := path.Join(dir, "prs.json")
+	log.Printf("saving metadata to %s", dest)
+	data, err := json.MarshalIndent(meta, "", "  ")
+	if err != nil {
+		return fmt.Errorf("error marshaling PR metadata: %w", err)
+	}
+	if err := os.WriteFile(dest, data, 0o644); err != nil {
+		return fmt.Errorf("error writing PR metadata file: %w", err)
+	}
+	return nil
+}
+
 func sendPrReport(version *semver.Version, prs []string, smtpPassword string) error {
 	log.Println("========================================================")
 	log.Println("The following PRs are created:")
@@ -436,12 +460,8 @@ func generateRepoList(
 			log.Printf("not bumping version on staging branch %s", branch)
 			continue
 		}
-		ok, err := fileExistsInGit(branch, versionFile)
-		if err != nil {
-			return []prRepo{}, fmt.Errorf("checking version file: %w", err)
-		}
-		if !ok {
-			log.Printf("skipping version bump on the %s branch, because %s does not exist on that branch", branch, versionFile)
+		if branch == fmt.Sprintf("release-%s-rc", releasedVersion.String()) {
+			log.Printf("not bumping version on the same branch %s", branch)
 			continue
 		}
 		curVersion, err := fileContent(remoteOrigin+"/"+branch, versionFile)
@@ -522,7 +542,7 @@ func generateRepoList(
 	// 5. Merge baking branch back to the release branch.
 	maybeBakingbranches := []string{
 		fmt.Sprintf("release-%s-rc", releasedVersion.String()), // e.g. release-23.1.17-rc
-		fmt.Sprintf("staging-%s", releasedVersion.Original()),  // e.g. staging-v23.1.17
+		fmt.Sprintf("staging-v%s", releasedVersion.String()),   // e.g. staging-v23.1.17
 	}
 	var bakingBranches []string
 	for _, branch := range maybeBakingbranches {
@@ -535,8 +555,23 @@ func generateRepoList(
 	if len(bakingBranches) > 1 {
 		return []prRepo{}, fmt.Errorf("too many baking branches: %s", strings.Join(maybeBakingbranches, ", "))
 	}
+	// 6. Merge baking branch to the main release branch (e.g. release-25.1).
+	// For pre-releases we may have no baking branches, thus we use `for` loop
+	// to simplify the code.
 	for _, mergeBranch := range bakingBranches {
 		baseBranch := fmt.Sprintf("release-%d.%d", releasedVersion.Major(), releasedVersion.Minor())
+		// Sometimes there are no changes on the baking/staging branches and a merge is not needed.
+		alreadyOnBaseBranch, err := isAncestor(mergeBranch, baseBranch)
+		if err != nil {
+			return []prRepo{}, fmt.Errorf("checking if %s is ancestor of %s: %w", baseBranch, mergeBranch, err)
+		}
+		if alreadyOnBaseBranch {
+			log.Printf("skipping merge of %s to %s, because %s is already an ancestor of %s", mergeBranch, baseBranch, mergeBranch, baseBranch)
+			continue
+		}
+		// TODO: add a check to make sure the merge generates no unexpected
+		// changes (we can ignore version.txt changes). The "ours" strategy
+		// doesn't account for changes in the merge branch.
 		repo := prRepo{
 			owner:          owner,
 			repo:           prefix + "cockroach",
@@ -545,7 +580,7 @@ func generateRepoList(
 			githubUsername: "cockroach-teamcity",
 			commitMessage:  generateCommitMessage(fmt.Sprintf("merge %s to %s", mergeBranch, baseBranch), releasedVersion, nextVersion),
 			fn: func(gitDir string) error {
-				cmd := exec.Command("git", "merge", "-s", "ours", "--no-commit", "origin/"+mergeBranch)
+				cmd := exec.Command("git", "merge", "-s", "ours", "--no-commit", remoteOrigin+"/"+mergeBranch)
 				cmd.Dir = gitDir
 				out, err := cmd.CombinedOutput()
 				if err != nil {
@@ -557,6 +592,68 @@ func generateRepoList(
 		}
 		reposToWorkOn = append(reposToWorkOn, repo)
 	}
+	// 7. Merge staging branch to the next release RC branch if it is present.
+	for _, mergeBranch := range bakingBranches {
+		// When we have extraordinary releases, we may have next release RC
+		// branches created. Make sure we merge this branch to the RC branch
+		// only if there are changes.
+		if !strings.HasPrefix(mergeBranch, "staging-") {
+			log.Printf("skipping merge of %s, because it's not a staging branch", mergeBranch)
+			continue
+		}
+		nextRCBranch := fmt.Sprintf("release-%s-rc", nextVersion.String())
+		maybeNextReleaseRCBranches, err := listRemoteBranches(nextRCBranch)
+		if err != nil {
+			return []prRepo{}, fmt.Errorf("listing rc branch %s: %w", nextRCBranch, err)
+		}
+		if len(maybeNextReleaseRCBranches) < 1 {
+			log.Printf("no next release RC branches found, skipping merge to %s", nextRCBranch)
+			continue
+		}
+		alreadyOnRCBranch, err := isAncestor(mergeBranch, nextRCBranch)
+		if err != nil {
+			return []prRepo{}, fmt.Errorf("checking if %s is ancestor of %s: %w", mergeBranch, nextRCBranch, err)
+		}
+		if alreadyOnRCBranch {
+			log.Printf("skipping merge of %s to %s, because %s is already an ancestor of %s", mergeBranch, nextRCBranch, mergeBranch, nextRCBranch)
+			continue
+		}
+		// try to merge and see if anything is changed, ignore version.txt changes.
+		createsMergeCommit, err := mergeCreatesContentChanges(mergeBranch, nextRCBranch, []string{versionFile})
+		if err != nil {
+			return []prRepo{}, fmt.Errorf("checking if merge creates content changes: %w", err)
+		}
+		if !createsMergeCommit {
+			log.Printf("skipping merge of %s to %s, because the merge does not create content changes", mergeBranch, nextRCBranch)
+			continue
+		}
+		repo := prRepo{
+			owner:          owner,
+			repo:           prefix + "cockroach",
+			branch:         nextRCBranch,
+			prBranch:       fmt.Sprintf("merge-%s-to-%s-%s", mergeBranch, nextRCBranch, randomString(4)),
+			githubUsername: "cockroach-teamcity",
+			commitMessage:  generateCommitMessage(fmt.Sprintf("merge %s to %s", mergeBranch, nextRCBranch), releasedVersion, nextVersion),
+			fn: func(gitDir string) error {
+				cmd := exec.Command("git", "merge", "-X", "ours", "--strategy=recursive", "--no-commit", "--no-ff", remoteOrigin+"/"+mergeBranch)
+				cmd.Dir = gitDir
+				out, err := cmd.CombinedOutput()
+				if err != nil {
+					return fmt.Errorf("failed running '%s' with message '%s': %w", cmd.String(), string(out), err)
+				}
+				log.Printf("ran '%s': %s\n", cmd.String(), string(out))
+				coCmd := exec.Command("git", "checkout", remoteOrigin+"/"+nextRCBranch, "--", versionFile)
+				coCmd.Dir = gitDir
+				out, err = coCmd.CombinedOutput()
+				if err != nil {
+					return fmt.Errorf("failed running '%s' with message '%s': %w", coCmd.String(), string(out), err)
+				}
+				log.Printf("ran '%s': %s\n", coCmd.String(), string(out))
+				return nil
+			},
+		}
+		reposToWorkOn = append(reposToWorkOn, repo)
+	}
 	return reposToWorkOn, nil
 }