Make release notes generation concurrent

Now that we retrieve the area labels from the PRs, the release notes
generation can take up to 2 minutes for big releases like 1.5. Although
in general this is not a problem since it should be a one time process,
it does slow down the comms release team when iterating in the notes,
adding missing labels, etc. It also makes development of the script
painful when testing it manually.

The default is 10 concurrent routines, but this can be changed with
command flag.

The default value of 10 doesn't seem to run into rate limiting problems.
I had to bump it to a 100 and run the command multiple times in a row to
be able to get rate limited.

Author: g-gaston

hack/tools/release/notes.go

@@ -31,6 +31,7 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -64,8 +65,9 @@ var (
 
 	fromTag = flag.String("from", "", "The tag or commit to start from.")
 
-	since = flag.String("since", "", "Include commits starting from and including this date. Accepts format: YYYY-MM-DD")
-	until = flag.String("until", "", "Include commits up to and including this date. Accepts format: YYYY-MM-DD")
+	since      = flag.String("since", "", "Include commits starting from and including this date. Accepts format: YYYY-MM-DD")
+	until      = flag.String("until", "", "Include commits up to and including this date. Accepts format: YYYY-MM-DD")
+	numWorkers = flag.Int("workers", 10, "Number of concurrent routines to process PR entries. If running into GitHub rate limiting, use 1.")
 
 	tagRegex = regexp.MustCompile(`^\[release-[\w-\.]*\]`)
 )
@@ -129,7 +131,7 @@ func getAreaLabel(merge string) (string, error) {
 
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("%s: %v", string(out), err)
 	}
 
 	pr := &githubPullRequest{}
@@ -223,53 +225,49 @@ func run() int {
 		}
 	}
 
-	for _, c := range commits {
-		body := trimTitle(c.body)
-		var key, prNumber, fork string
-		prefix, err := getAreaLabel(c.merge)
-		if err != nil {
-			fmt.Println(err)
-			os.Exit(1)
-		}
-		switch {
-		case strings.HasPrefix(body, ":sparkles:"), strings.HasPrefix(body, "✨"):
-			key = features
-			body = strings.TrimPrefix(body, ":sparkles:")
-			body = strings.TrimPrefix(body, "✨")
-		case strings.HasPrefix(body, ":bug:"), strings.HasPrefix(body, "🐛"):
-			key = bugs
-			body = strings.TrimPrefix(body, ":bug:")
-			body = strings.TrimPrefix(body, "🐛")
-		case strings.HasPrefix(body, ":book:"), strings.HasPrefix(body, "📖"):
-			key = documentation
-			body = strings.TrimPrefix(body, ":book:")
-			body = strings.TrimPrefix(body, "📖")
-			if strings.Contains(body, "CAEP") || strings.Contains(body, "proposal") {
-				key = proposals
+	results := make(chan releaseNoteEntryResult)
+	commitCh := make(chan *commit)
+	var wg sync.WaitGroup
+
+	wg.Add(*numWorkers)
+	for i := 0; i < *numWorkers; i++ {
+		go func() {
+			for commit := range commitCh {
+				processed := releaseNoteEntryResult{}
+				processed.prEntry, processed.err = generateReleaseNoteEntry(commit)
+				results <- processed
 			}
-		case strings.HasPrefix(body, ":seedling:"), strings.HasPrefix(body, "🌱"):
-			key = other
-			body = strings.TrimPrefix(body, ":seedling:")
-			body = strings.TrimPrefix(body, "🌱")
-		case strings.HasPrefix(body, ":warning:"), strings.HasPrefix(body, "⚠️"):
-			key = warning
-			body = strings.TrimPrefix(body, ":warning:")
-			body = strings.TrimPrefix(body, "⚠️")
-		default:
-			key = unknown
+			wg.Done()
+		}()
+	}
+
+	go func() {
+		for _, c := range commits {
+			commitCh <- c
+		}
+		close(commitCh)
+	}()
+
+	go func() {
+		wg.Wait()
+		close(results)
+	}()
+
+	for result := range results {
+		if result.err != nil {
+			fmt.Println(result.err)
+			os.Exit(0)
 		}
 
-		body = strings.TrimSpace(body)
-		if body == "" {
+		if result.prEntry.title == "" {
 			continue
 		}
-		body = fmt.Sprintf("- %s: %s", prefix, body)
-		_, _ = fmt.Sscanf(c.merge, "Merge pull request %s from %s", &prNumber, &fork)
-		if key == documentation {
-			merges[key] = append(merges[key], prNumber)
-			continue
+
+		if result.prEntry.section == documentation {
+			merges[result.prEntry.section] = append(merges[result.prEntry.section], result.prEntry.prNumber)
+		} else {
+			merges[result.prEntry.section] = append(merges[result.prEntry.section], result.prEntry.title)
 		}
-		merges[key] = append(merges[key], formatMerge(body, prNumber))
 	}
 
 	// TODO Turn this into a link (requires knowing the project name + organization)
@@ -346,3 +344,66 @@ func commandExists(cmd string) bool {
 	_, err := exec.LookPath(cmd)
 	return err == nil
 }
+
+// releaseNoteEntryResult is the result of processing a PR to create a release note item.
+// Used to aggregate the line item and error when processing concurrently.
+type releaseNoteEntryResult struct {
+	prEntry *releaseNoteEntry
+	err     error
+}
+
+// releaseNoteEntry represents a line item in the release notes.
+type releaseNoteEntry struct {
+	title    string
+	section  string
+	prNumber string
+}
+
+// generateReleaseNoteEntry processes a commit into a PR line item for the release notes.
+func generateReleaseNoteEntry(c *commit) (*releaseNoteEntry, error) {
+	entry := &releaseNoteEntry{}
+	entry.title = trimTitle(c.body)
+	var fork string
+	prefix, err := getAreaLabel(c.merge)
+	if err != nil {
+		return nil, err
+	}
+
+	switch {
+	case strings.HasPrefix(entry.title, ":sparkles:"), strings.HasPrefix(entry.title, "✨"):
+		entry.section = features
+		entry.title = strings.TrimPrefix(entry.title, ":sparkles:")
+		entry.title = strings.TrimPrefix(entry.title, "✨")
+	case strings.HasPrefix(entry.title, ":bug:"), strings.HasPrefix(entry.title, "🐛"):
+		entry.section = bugs
+		entry.title = strings.TrimPrefix(entry.title, ":bug:")
+		entry.title = strings.TrimPrefix(entry.title, "🐛")
+	case strings.HasPrefix(entry.title, ":book:"), strings.HasPrefix(entry.title, "📖"):
+		entry.section = documentation
+		entry.title = strings.TrimPrefix(entry.title, ":book:")
+		entry.title = strings.TrimPrefix(entry.title, "📖")
+		if strings.Contains(entry.title, "CAEP") || strings.Contains(entry.title, "proposal") {
+			entry.section = proposals
+		}
+	case strings.HasPrefix(entry.title, ":seedling:"), strings.HasPrefix(entry.title, "🌱"):
+		entry.section = other
+		entry.title = strings.TrimPrefix(entry.title, ":seedling:")
+		entry.title = strings.TrimPrefix(entry.title, "🌱")
+	case strings.HasPrefix(entry.title, ":warning:"), strings.HasPrefix(entry.title, "⚠️"):
+		entry.section = warning
+		entry.title = strings.TrimPrefix(entry.title, ":warning:")
+		entry.title = strings.TrimPrefix(entry.title, "⚠️")
+	default:
+		entry.section = unknown
+	}
+
+	entry.title = strings.TrimSpace(entry.title)
+	if entry.title == "" {
+		return entry, nil
+	}
+	entry.title = fmt.Sprintf("- %s: %s", prefix, entry.title)
+	_, _ = fmt.Sscanf(c.merge, "Merge pull request %s from %s", &entry.prNumber, &fork)
+	entry.title = formatMerge(entry.title, entry.prNumber)
+
+	return entry, nil
+}