Merge #142495 #143330

142495: ui: upgrade d3 to latest version r=kyle-a-wong a=kyle-a-wong

replaced the usage of the d3 package with only the
d3-* modules that we use in cluster-ui / db-console
and upgraded to the latest versions of each of these
packages.

Changes:

- This commit includes an updated world.json geojson
file used to drop the world for the overview node map.
For unknown reasons, the previous version of the geojson
didn't render the united states, random parts of africa,
and a few other countries. This is semi-custom because
I wasn't able to find a source for a version that included
countries AND us state borders. To create this file, I
downloaded a world geo json file and a a united states
geo json file and merged the feature collections. I also
edited the id fields of the stats to include the
abbreviations of each state to replace numerical ids.

- The capacity chart was updated to properly render
the capacity chart in the overview page. The main change
involves adding a call to .merge() to ensure that the d3
components are properly rendered on the screen the first
time they are appeneded to the main svg. See:
https://github.com/d3/d3/blob/main/CHANGES.md#selections-d3-selection
for more details.

- Some zoom behavior functionality changed, and zoom transform is
now available on event.transform. mapLayout.tsx was updated to
account for these changes. See:
https://github.com/d3/d3/blob/main/CHANGES.md#zooming-d3-zoom
for more details.

- nestedLocations was reimplemente to use array.reduce() and no longer
uses d3's nest function

- replaced d3.keys with Object.keys

- Updated db-console babel.config and jest.config to match cluster-ui's
corresponding configs

Resolves: [CRDB-48219](https://cockroachlabs.atlassian.net/browse/CRDB-48219)
Epic: [CRDB-32154](https://cockroachlabs.atlassian.net/browse/CRDB-32154)
Release note: None

143330: release: merge ER commits to RC branches r=celiala a=rail

Previously, as a part of post-publishing tasks, we created merge commits to the main release branch. When we have an extraordinary release, there is a chance of having the next release's branch already cut. If the extraordinary release PRs are not backported to the next release's branch, we may have a regression, when the next version doesn't have the fix merged to the extraordinary release branch.

This PR adds an additional check to verify that all changes from the extraordinary branch are backported to the next release branch if it's already cut. If there are changes, automation creates a merge PR to have all extraordinary changes in the next release branch.

Additionally, add a check if there were no changes on the baking/staging branch after it was cut and skip the merge PR creation.

Added a global flag pointing to the artifacts directory for cases we need to store some outputs.

Fixes: RE-817
Release note: None

Co-authored-by: Kyle Wong <[email protected]>
Co-authored-by: Rail Aliiev <[email protected]>

Author: 3 people

build/teamcity/internal/cockroach/release/process/update_versions_impl.sh

@@ -37,4 +37,5 @@ $(bazel info --config=crosslinux bazel-bin)/pkg/cmd/release/release_/release \
   [email protected] \
   --smtp-host=smtp.gmail.com \
   --smtp-port=587 \
+  --artifacts-dir=/artifacts \
   --to=$to

pkg/cmd/release/git.go

@@ -7,6 +7,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log"
 	"os/exec"
@@ -288,7 +289,7 @@ func listRemoteBranches(pattern string) ([]string, error) {
 	if err != nil {
 		return []string{}, fmt.Errorf("git ls-remote: %w", err)
 	}
-	log.Printf("git ls-remote returned: %s", out)
+	log.Printf("git ls-remote for %s returned: %s", pattern, out)
 	var remoteBranches []string
 	// Example output:
 	// $ git ls-remote origin "refs/heads/release-23.1*"
@@ -307,25 +308,102 @@ func listRemoteBranches(pattern string) ([]string, error) {
 	return remoteBranches, nil
 }
 
-// fileExistsInGit checks if a file exists in a local repository, assuming the remote name is `origin`.
-func fileExistsInGit(branch string, f string) (bool, error) {
-	cmd := exec.Command("git", "ls-tree", remoteOrigin+"/"+branch, f)
+// fileContent uses `git cat-file -p ref:file` to get to the file contents without `git checkout`.
+func fileContent(ref string, f string) (string, error) {
+	cmd := exec.Command("git", "cat-file", "-p", ref+":"+f)
 	out, err := cmd.Output()
 	if err != nil {
-		return false, fmt.Errorf("git ls-tree: %s %s %w, `%s`", branch, f, err, out)
+		return "", fmt.Errorf("git cat-file %s:%s: %w, `%s`", ref, f, err, out)
 	}
-	if len(out) == 0 {
-		return false, nil
+	return string(out), nil
+}
+
+// isAncestor checks if ref1 is an ancestor of ref2.
+// Returns true if ref1 is an ancestor of ref2, false if not, and error if the command fails.
+func isAncestor(ref1, ref2 string) (bool, error) {
+	cmd := exec.Command("git", "merge-base", "--is-ancestor", remoteOrigin+"/"+ref1, remoteOrigin+"/"+ref2)
+	err := cmd.Run()
+	if err != nil {
+		// Treat exit code 1 as false, as it means that ref1 is not an ancestor of ref2.
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 {
+			return false, nil
+		}
+		return false, fmt.Errorf("checking ancestry relationship between %s and %s: %w", ref1, ref2, err)
 	}
 	return true, nil
 }
 
-// fileContent uses `git cat-file -p ref:file` to get to the file contents without `git checkout`.
-func fileContent(ref string, f string) (string, error) {
-	cmd := exec.Command("git", "cat-file", "-p", ref+":"+f)
-	out, err := cmd.Output()
+// mergeCreatesContentChanges checks if a merge commit introduces changes to the branch.
+// Returns true if the merge commit introduces changes, false if not, and error if the command fails.
+func mergeCreatesContentChanges(branch, intoBranch string, ignoredPatterns []string) (bool, error) {
+	// Make sure the working directory is clean
+	statusCmd := exec.Command("git", "status", "--porcelain")
+	if out, err := statusCmd.Output(); err != nil {
+		return false, fmt.Errorf("checking git status: %w", err)
+	} else if len(out) > 0 {
+		return false, fmt.Errorf("working directory is not clean")
+	}
+
+	// Get the current branch name before we start
+	currentBranchCmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD")
+	currentBranch, err := currentBranchCmd.Output()
 	if err != nil {
-		return "", fmt.Errorf("git cat-file %s:%s: %w, `%s`", ref, f, err, out)
+		return false, fmt.Errorf("getting current branch: %w", err)
 	}
-	return string(out), nil
+	originalBranch := strings.TrimSpace(string(currentBranch))
+
+	// Checkout the branch to merge into. Use a temporary branch to avoid
+	// conflicts with the current branch name.
+	tmpIntoBranch := intoBranch + "-tmp"
+	checkoutCmd := exec.Command("git", "checkout", "-b", tmpIntoBranch, remoteOrigin+"/"+intoBranch)
+	if err := checkoutCmd.Run(); err != nil {
+		return false, fmt.Errorf("running checkout: %w", err)
+	}
+
+	// Run the merge command without committing and without fast-forward. If
+	// fast-forward is allowed and the current branch can fast forward, there
+	// will be no merge commit, so the --no-commit option won't work.
+	// We need to use the ours strategy to avoid conflicts. In the next step we
+	// will checkout the ignored files from the current branch (like version.txt).
+	mergeCmd := exec.Command("git", "merge", "--no-commit", "--no-ff", "--strategy=recursive", "-X", "ours", remoteOrigin+"/"+branch)
+	if err := mergeCmd.Run(); err != nil {
+		return false, fmt.Errorf("running merge: %w", err)
+	}
+	if len(ignoredPatterns) > 0 {
+		coCmd := exec.Command("git", "checkout", tmpIntoBranch, "--")
+		coCmd.Args = append(coCmd.Args, ignoredPatterns...)
+
+		if err := coCmd.Run(); err != nil {
+			return false, fmt.Errorf("running checkout: %w", err)
+		}
+	}
+
+	// Check if there are any content changes. The exit code will be analyzed to
+	// determine if there are changes after we clean up the current repo.
+	diffCmd := exec.Command("git", "diff", "--staged", "--quiet")
+	diffErr := diffCmd.Run()
+
+	// Always abort the merge attempt to clean up
+	if err := exec.Command("git", "merge", "--abort").Run(); err != nil {
+		return false, fmt.Errorf("aborting merge: %w", err)
+	}
+	if err := exec.Command("git", "checkout", originalBranch).Run(); err != nil {
+		return false, fmt.Errorf("running original branch checkout: %w", err)
+	}
+	if err := exec.Command("git", "branch", "-D", tmpIntoBranch).Run(); err != nil {
+		return false, fmt.Errorf("deleting tmp branch: %w", err)
+	}
+
+	// If diff returns no error (exit code 0), there are no changes
+	// If diff returns error with exit code 1, there are changes
+	// Any other error is unexpected
+	if diffErr == nil {
+		return false, nil // No changes
+	}
+	var exitErr *exec.ExitError
+	if errors.As(diffErr, &exitErr) && exitErr.ExitCode() == 1 {
+		return true, nil // Has changes
+	}
+	return false, fmt.Errorf("checking diff: %w", diffErr)
 }

pkg/cmd/release/main.go

@@ -8,6 +8,11 @@ package main
 import "github.com/spf13/cobra"
 
 var rootCmd = &cobra.Command{Use: "release"}
+var artifactsDir string
+
+func init() {
+	rootCmd.PersistentFlags().StringVar(&artifactsDir, "artifacts-dir", "", "artifacts directory")
+}
 
 const (
 	envSMTPUser     = "SMTP_USER"

pkg/cmd/release/update_versions.go

@@ -102,6 +102,10 @@ type prRepo struct {
 	workOnRepoError error
 }
 
+type metadata struct {
+	PRs []string `json:"prs"`
+}
+
 func (r prRepo) String() string {
 	return r.owner + "/" + r.repo + "@" + r.branch
 }
@@ -335,12 +339,32 @@ func updateVersions(_ *cobra.Command, _ []string) error {
 		workOnRepoErrors = append(workOnRepoErrors, err)
 		log.Printf("%s", err)
 	}
+	if artifactsDir != "" {
+		if err := saveMetadata(artifactsDir, metadata{PRs: prs}); err != nil {
+			err = fmt.Errorf("error saving metadata: %w", err)
+			workOnRepoErrors = append(workOnRepoErrors, err)
+			log.Printf("%s", err)
+		}
+	}
 	if len(workOnRepoErrors) > 0 {
 		return errors.Join(workOnRepoErrors...)
 	}
 	return nil
 }
 
+func saveMetadata(dir string, meta metadata) error {
+	dest := path.Join(dir, "prs.json")
+	log.Printf("saving metadata to %s", dest)
+	data, err := json.MarshalIndent(meta, "", "  ")
+	if err != nil {
+		return fmt.Errorf("error marshaling PR metadata: %w", err)
+	}
+	if err := os.WriteFile(dest, data, 0o644); err != nil {
+		return fmt.Errorf("error writing PR metadata file: %w", err)
+	}
+	return nil
+}
+
 func sendPrReport(version *semver.Version, prs []string, smtpPassword string) error {
 	log.Println("========================================================")
 	log.Println("The following PRs are created:")
@@ -436,12 +460,8 @@ func generateRepoList(
 			log.Printf("not bumping version on staging branch %s", branch)
 			continue
 		}
-		ok, err := fileExistsInGit(branch, versionFile)
-		if err != nil {
-			return []prRepo{}, fmt.Errorf("checking version file: %w", err)
-		}
-		if !ok {
-			log.Printf("skipping version bump on the %s branch, because %s does not exist on that branch", branch, versionFile)
+		if branch == fmt.Sprintf("release-%s-rc", releasedVersion.String()) {
+			log.Printf("not bumping version on the same branch %s", branch)
 			continue
 		}
 		curVersion, err := fileContent(remoteOrigin+"/"+branch, versionFile)
@@ -522,7 +542,7 @@ func generateRepoList(
 	// 5. Merge baking branch back to the release branch.
 	maybeBakingbranches := []string{
 		fmt.Sprintf("release-%s-rc", releasedVersion.String()), // e.g. release-23.1.17-rc
-		fmt.Sprintf("staging-%s", releasedVersion.Original()),  // e.g. staging-v23.1.17
+		fmt.Sprintf("staging-v%s", releasedVersion.String()),   // e.g. staging-v23.1.17
 	}
 	var bakingBranches []string
 	for _, branch := range maybeBakingbranches {
@@ -535,8 +555,23 @@ func generateRepoList(
 	if len(bakingBranches) > 1 {
 		return []prRepo{}, fmt.Errorf("too many baking branches: %s", strings.Join(maybeBakingbranches, ", "))
 	}
+	// 6. Merge baking branch to the main release branch (e.g. release-25.1).
+	// For pre-releases we may have no baking branches, thus we use `for` loop
+	// to simplify the code.
 	for _, mergeBranch := range bakingBranches {
 		baseBranch := fmt.Sprintf("release-%d.%d", releasedVersion.Major(), releasedVersion.Minor())
+		// Sometimes there are no changes on the baking/staging branches and a merge is not needed.
+		alreadyOnBaseBranch, err := isAncestor(mergeBranch, baseBranch)
+		if err != nil {
+			return []prRepo{}, fmt.Errorf("checking if %s is ancestor of %s: %w", baseBranch, mergeBranch, err)
+		}
+		if alreadyOnBaseBranch {
+			log.Printf("skipping merge of %s to %s, because %s is already an ancestor of %s", mergeBranch, baseBranch, mergeBranch, baseBranch)
+			continue
+		}
+		// TODO: add a check to make sure the merge generates no unexpected
+		// changes (we can ignore version.txt changes). The "ours" strategy
+		// doesn't account for changes in the merge branch.
 		repo := prRepo{
 			owner:          owner,
 			repo:           prefix + "cockroach",
@@ -545,7 +580,7 @@ func generateRepoList(
 			githubUsername: "cockroach-teamcity",
 			commitMessage:  generateCommitMessage(fmt.Sprintf("merge %s to %s", mergeBranch, baseBranch), releasedVersion, nextVersion),
 			fn: func(gitDir string) error {
-				cmd := exec.Command("git", "merge", "-s", "ours", "--no-commit", "origin/"+mergeBranch)
+				cmd := exec.Command("git", "merge", "-s", "ours", "--no-commit", remoteOrigin+"/"+mergeBranch)
 				cmd.Dir = gitDir
 				out, err := cmd.CombinedOutput()
 				if err != nil {
@@ -557,6 +592,68 @@ func generateRepoList(
 		}
 		reposToWorkOn = append(reposToWorkOn, repo)
 	}
+	// 7. Merge staging branch to the next release RC branch if it is present.
+	for _, mergeBranch := range bakingBranches {
+		// When we have extraordinary releases, we may have next release RC
+		// branches created. Make sure we merge this branch to the RC branch
+		// only if there are changes.
+		if !strings.HasPrefix(mergeBranch, "staging-") {
+			log.Printf("skipping merge of %s, because it's not a staging branch", mergeBranch)
+			continue
+		}
+		nextRCBranch := fmt.Sprintf("release-%s-rc", nextVersion.String())
+		maybeNextReleaseRCBranches, err := listRemoteBranches(nextRCBranch)
+		if err != nil {
+			return []prRepo{}, fmt.Errorf("listing rc branch %s: %w", nextRCBranch, err)
+		}
+		if len(maybeNextReleaseRCBranches) < 1 {
+			log.Printf("no next release RC branches found, skipping merge to %s", nextRCBranch)
+			continue
+		}
+		alreadyOnRCBranch, err := isAncestor(mergeBranch, nextRCBranch)
+		if err != nil {
+			return []prRepo{}, fmt.Errorf("checking if %s is ancestor of %s: %w", mergeBranch, nextRCBranch, err)
+		}
+		if alreadyOnRCBranch {
+			log.Printf("skipping merge of %s to %s, because %s is already an ancestor of %s", mergeBranch, nextRCBranch, mergeBranch, nextRCBranch)
+			continue
+		}
+		// try to merge and see if anything is changed, ignore version.txt changes.
+		createsMergeCommit, err := mergeCreatesContentChanges(mergeBranch, nextRCBranch, []string{versionFile})
+		if err != nil {
+			return []prRepo{}, fmt.Errorf("checking if merge creates content changes: %w", err)
+		}
+		if !createsMergeCommit {
+			log.Printf("skipping merge of %s to %s, because the merge does not create content changes", mergeBranch, nextRCBranch)
+			continue
+		}
+		repo := prRepo{
+			owner:          owner,
+			repo:           prefix + "cockroach",
+			branch:         nextRCBranch,
+			prBranch:       fmt.Sprintf("merge-%s-to-%s-%s", mergeBranch, nextRCBranch, randomString(4)),
+			githubUsername: "cockroach-teamcity",
+			commitMessage:  generateCommitMessage(fmt.Sprintf("merge %s to %s", mergeBranch, nextRCBranch), releasedVersion, nextVersion),
+			fn: func(gitDir string) error {
+				cmd := exec.Command("git", "merge", "-X", "ours", "--strategy=recursive", "--no-commit", "--no-ff", remoteOrigin+"/"+mergeBranch)
+				cmd.Dir = gitDir
+				out, err := cmd.CombinedOutput()
+				if err != nil {
+					return fmt.Errorf("failed running '%s' with message '%s': %w", cmd.String(), string(out), err)
+				}
+				log.Printf("ran '%s': %s\n", cmd.String(), string(out))
+				coCmd := exec.Command("git", "checkout", remoteOrigin+"/"+nextRCBranch, "--", versionFile)
+				coCmd.Dir = gitDir
+				out, err = coCmd.CombinedOutput()
+				if err != nil {
+					return fmt.Errorf("failed running '%s' with message '%s': %w", coCmd.String(), string(out), err)
+				}
+				log.Printf("ran '%s': %s\n", coCmd.String(), string(out))
+				return nil
+			},
+		}
+		reposToWorkOn = append(reposToWorkOn, repo)
+	}
 	return reposToWorkOn, nil
 }